diff --git a/cl_query_data_job/pom.xml b/cl_query_data_job/pom.xml index d66c17a..38f10c1 100644 --- a/cl_query_data_job/pom.xml +++ b/cl_query_data_job/pom.xml @@ -3,10 +3,11 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + - cl_stream_3.2 + cl_stream_3.3 com.bfd.mf - 3.2-SNAPSHOT + 3.3-SNAPSHOT cl_query_data_job diff --git a/cl_search_api/pom.xml b/cl_search_api/pom.xml index 8d8893a..6f6d543 100644 --- a/cl_search_api/pom.xml +++ b/cl_search_api/pom.xml @@ -10,24 +10,27 @@ 3.3-SNAPSHOT + cl_search_api Search V3.3 API cl_search_api 3.3.0-SNAPSHOT + + com.bfd.mf.SearchApplication 1.8 6.0.0 2.0.0.RELEASE 2.0.0.RELEASE - 5.0.4.RELEASE + 5.0.16.RELEASE 4.4 - + 4.1.0 9.4.8.v20171121 1.1.6 19.0 - 3.15 + 4.1.0 1.8 UTF-8 UTF-8 @@ -90,7 +93,7 @@ org.mybatis.spring.boot mybatis-spring-boot-starter - 1.3.1 + 2.2.2 org.springframework.boot @@ -124,7 +127,7 @@ com.alibaba fastjson - 1.2.68 + 1.2.83 @@ -133,6 +136,12 @@ 1.16.20 + + + + + + mysql mysql-connector-java @@ -166,22 +175,22 @@ com.fasterxml.jackson.core jackson-databind - 2.9.6 + 2.9.9.2 - - - + + + - - - + + + - - - + + + net.logstash.logback @@ -199,9 +208,9 @@ 1.1.7 - - - + + + io.springfox @@ -227,9 +236,9 @@ - - - + + + com.google.guava @@ -237,16 +246,16 @@ 25.0-jre - - org.apache.poi - poi - 4.1.0 - - - org.apache.poi - poi-ooxml - 4.1.0 - + + + + + + + + + + @@ -254,26 +263,19 @@ okhttp 3.6.0 - - commons-lang - commons-lang - 2.6 - compile - - - - - org.jsoup - jsoup - 1.10.2 - - - - + + + + + + + + + diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java index 9044e61..e7a8b40 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java @@ -17,7 +17,6 @@ package com.bfd.mf.common.util.constants; import com.bfd.mf.common.util.utility.EsQueryConditionUtils; -import com.bfd.mf.config.BFDApiConfig; import java.util.ArrayList; import java.util.HashMap; @@ -729,6 +728,7 @@ public class ESConstant { ESConstant.AUTHOR, //用户名 ESConstant.AUTHORNICKNAME, // 用户昵称 ESConstant.AUTHORID, // 用户ID + ESConstant.USER_URL, ESConstant.DOC_ID ,// 主贴唯一ID ESConstant.DATA_ID, // 数据唯一ID @@ -740,18 +740,23 @@ public class ESConstant { ESConstant.CRAWLTIME , // 抓取时间 ESConstant.PUBTIME , // 发表时间 + ESConstant.PUBTIMESTR, + ESConstant.CRAWLTIMESTR, + ESConstant.QUOTE_COUNT , // 转发数 ESConstant.COMMENTS_COUNT , // 评论数 ESConstant.ATTITUDES_COUNT , // 点赞数 ESConstant.PRICE, // 价格 ESConstant.POST_COUNT, // 销量 + ESConstant.COLLE_CTCOUNT, // 收藏数 + ESConstant.VIEW_CNT, // 浏览数 ESConstant.TITLE ,// 标题 ESConstant.CONTENT , // 正文 ESConstant.TRANSLATETITLE, // 译文标题 ESConstant.TRANSLATECONTENT, // 译文正文 - ESConstant.FORWARD_CONTENT, // 正文源码 + /* ESConstant.FORWARD_CONTENT, // 正文源码*/ ESConstant.SYS_SENTIMENT, // 情感 ESConstant.PRIMARY, // 数据类型 @@ -818,7 +823,7 @@ public class ESConstant { ESConstant.ISDOWNLOAD, // 是否下载 ESConstant.FILEPATH, // 文件路径 - ESConstant.IMAGEPATH, // 图片路径 + ESConstant.IMAGEPATH, // 图 片路径 ESConstant.VIDEOPATH, // 视频路径 ESConstant.FILEPATHSIZE, // 文件详细路径 ESConstant.IMAGEPATHSIZE, // 图片详细路径 @@ -843,7 +848,7 @@ public class ESConstant { ); // 总体分析要用的字段 - public static final String[] FIELD_LIST_ANALYSIS = + public static final String[] FIELD_LIST_ANALYSIS = EsQueryConditionUtils.getIncludeQueryField( ESConstant.PUBTIME , ESConstant.CRAWLTIME diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java index c3a8daf..2ce9e10 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java @@ -287,7 +287,7 @@ public abstract class EsUtils { private static void getHighlightResult(String fieldName, SearchHit hit, JSONObject data) { if (hit.getHighlightFields().containsKey(fieldName)) { HighlightField highlightField = hit.getHighlightFields().get(fieldName); - System.out.println("getHighlightResult highlightField : "+highlightField); + System.out.println("getHighlightResult highlightField : " + highlightField); Text[] fragments = highlightField.fragments(); String fragmentString = ""; for (Text fragment : fragments) { @@ -511,6 +511,7 @@ public abstract class EsUtils { .setQuery(boolQueryBuilder) .setSearchType(SearchType.DEFAULT) .setSize(limit) + //.setFetchSource(ESConstant.FIELD_DATA, null) .setScroll(new TimeValue(300000)) .execute() .actionGet();//注意:首次搜索并不包含数据 diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchAnalysisService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchAnalysisService.java index bca2c05..0497cd8 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchAnalysisService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchAnalysisService.java @@ -41,7 +41,8 @@ public class SearchAnalysisService { JSONObject jsonObject = new JSONObject(); try{ if(null != queryRequest.getSubjectId()) { - List esMonitorEntity = sliceScrollUtil.fetchResultSubjectCache(queryRequest, ESConstant.FIELD_LIST_ANALYSIS); + List esMonitorEntity = sliceScrollUtil.fetchResultSubjectCache(queryRequest, + ESConstant.FIELD_LIST_ANALYSIS); // 渠道走势 jsonObject = dataAnalysisTrendByDayQueryTimes(queryRequest, esMonitorEntity); // 获取 渠道统计结果 分类标签统计结果 价值标签统计结果 diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java index 1063066..70f68f3 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java @@ -574,19 +574,21 @@ public class SearchDataService extends CrudService")){ // Document doc = Jsoup.parse(forwardContent); //String text = Jsoup.clean(forwardContent, Whitelist.basicWithImages()); String text = Jsoup.clean(forwardContent, Whitelist.basic()); - forContent = text; - // } esMonitorEntity.setForwardContent(forContent); + + + esMonitorEntity.setForwardContent(""); esMonitorEntity.setReadCount(readCount); esMonitorEntity.setHasFile(hasFile); esMonitorEntity.setHasVideo(hasVideo);