From 3d82868826f5b33109ef9848a2f74996645b65ba Mon Sep 17 00:00:00 2001 From: "jing.du" Date: Thu, 21 Apr 2022 17:12:01 +0800 Subject: [PATCH] =?UTF-8?q?20220421=E7=89=88=E6=9C=AC-=E7=B2=BE=E7=AE=80?= =?UTF-8?q?=E4=BA=86=E4=B8=80=E4=B8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/compiler.xml | 2 +- .idea/modules.xml | 2 +- cl_query_data_job/pom.xml | 5 +- cl_search_api/cl_search_api.iml | 4 +- cl_search_api/pom.xml | 9 +- .../mf/common/service/cache/TopicQueryService.java | 250 +++++++---- .../bfd/mf/common/service/common/CrudService.java | 4 +- .../bfd/mf/common/service/es/ClusterService.java | 4 +- .../bfd/mf/common/service/es/EsCommonService.java | 7 +- .../service/es/EsQueryAuthorCountService.java | 2 - .../mf/common/service/es/EsQueryAuthorService.java | 23 +- .../common/service/es/EsQueryServiceForSQMini.java | 75 +++- .../service/es/EsQueryServiceForSQNormal.java | 2 +- .../bfd/mf/common/service/es/GetQueryBuilder.java | 12 +- .../common/service/es/ParseSearchScopeService.java | 2 +- .../common/service/es/SubjectQueryDataService.java | 5 +- .../bfd/mf/common/service/text/TextService.java | 2 - .../mf/common/util/analysis/DataAnalysisUtils.java | 4 +- .../mf/common/util/cache/ResponseParseUtils.java | 1 - .../bfd/mf/common/util/constants/ESConstant.java | 13 +- .../java/com/bfd/mf/common/util/es/EsUtils.java | 261 +++++++++--- .../java/com/bfd/mf/common/util/es/EsUtils2.java | 2 - .../com/bfd/mf/common/util/es/MonitorUtils.java | 3 - .../bfd/mf/common/util/slice/SliceScrollUtil.java | 10 +- .../com/bfd/mf/common/util/utility/DateUtil.java | 2 - .../mf/common/web/entity/mysql/cache/Cluster.java | 46 +- .../web/entity/mysql/topic/ParseExcelTask.java | 1 - .../web/repository/mysql/UserRepository.java | 469 --------------------- .../mysql/topic/ParseExcelTaskRepository.java | 1 - .../bfd/mf/common/web/vo/params/QueryRequest.java | 2 +- .../common/web/vo/view/analysis/DataLineCount.java | 88 ---- .../web/vo/view/monitor/ESMonitorBaseEntity.java | 145 +++++-- .../web/vo/view/monitor/ESMonitorEntity.java | 2 - .../mf/controller/SearchAnalysisController.java | 9 +- .../bfd/mf/controller/SearchAuthorController.java | 3 +- .../bfd/mf/controller/SearchDataController.java | 127 +++++- .../bfd/mf/controller/UploadExcelController.java | 8 +- .../com/bfd/mf/service/SearchAnalysisService.java | 9 +- .../com/bfd/mf/service/SearchAuthorService.java | 5 +- .../java/com/bfd/mf/service/SearchDataService.java | 271 ++++++++---- .../bfd/mf/service/SearchKeywordsCouldService.java | 18 +- .../java/com/bfd/mf/service/UpdateService.java | 8 +- .../com/bfd/mf/service/UploadExcelService.java | 11 +- cl_stream_3.1.2.iml | 12 - pom.xml | 4 +- 45 files changed, 987 insertions(+), 958 deletions(-) delete mode 100644 cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/UserRepository.java delete mode 100644 cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataLineCount.java delete mode 100644 cl_stream_3.1.2.iml diff --git a/.idea/compiler.xml b/.idea/compiler.xml index f85b716..a50c87d 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -14,8 +14,8 @@ + - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml index c7f9e39..921be69 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -4,7 +4,7 @@ - + \ No newline at end of file diff --git a/cl_query_data_job/pom.xml b/cl_query_data_job/pom.xml index 7725c64..4f9f20f 100644 --- a/cl_query_data_job/pom.xml +++ b/cl_query_data_job/pom.xml @@ -4,9 +4,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - cl_stream_3.1.2 + cl_stream_3.2 com.bfd.mf - 3.1.2-SNAPSHOT + 3.2-SNAPSHOT cl_query_data_job @@ -232,6 +232,7 @@ org.springframework.boot spring-boot-maven-plugin + 2.0.0.RELEASE com.bfd.mf.job.Application diff --git a/cl_search_api/cl_search_api.iml b/cl_search_api/cl_search_api.iml index e141da7..dcb9b34 100644 --- a/cl_search_api/cl_search_api.iml +++ b/cl_search_api/cl_search_api.iml @@ -14,7 +14,9 @@ - + + + diff --git a/cl_search_api/pom.xml b/cl_search_api/pom.xml index 922467e..fad6989 100644 --- a/cl_search_api/pom.xml +++ b/cl_search_api/pom.xml @@ -5,15 +5,15 @@ 4.0.0 - cl_stream_3.1.2 + cl_stream_3.2 com.bfd.mf - 3.1.2-SNAPSHOT + 3.2-SNAPSHOT cl_search_api - Search V3.1.2 API + Search V3.2 API cl_search_api - 3.1.2-SNAPSHOT + 3.2-SNAPSHOT @@ -278,6 +278,7 @@ org.springframework.boot spring-boot-maven-plugin + 2.0.0.RELEASE com.bfd.mf.SearchApplication diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/cache/TopicQueryService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/cache/TopicQueryService.java index 105debb..a20b442 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/cache/TopicQueryService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/cache/TopicQueryService.java @@ -9,17 +9,23 @@ import com.bfd.mf.common.web.repository.mysql.base.SiteRepository; import com.bfd.mf.common.web.repository.mysql.topic.TaskRepository; import com.bfd.mf.common.web.vo.params.QueryRequest; import com.bfd.nlp.common.util.object.TObjectUtils; -import org.apache.log4j.Logger; -import org.elasticsearch.index.query.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @Service public class TopicQueryService { - private static Logger logger = Logger.getLogger(TopicQueryService.class); + private static Logger logger = LoggerFactory.getLogger(TopicQueryService.class); @Autowired private EsCommonService esCommonService; @Autowired @@ -32,28 +38,28 @@ public class TopicQueryService { * 条件查询语句组装 渠道、站点、地区、时间、任务标识等 */ public BoolQueryBuilder queryByConditions_v1(QueryRequest queryRequest) { - logger.info("[TopicQueryService] queryByConditions_v1 " ); + logger.info("[TopicQueryService] queryByConditions_v1 "); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); try { // 价值标签 - if(!("").equals(queryRequest.getValueLabel()) && null != queryRequest.getValueLabel()){ - QueryBuilder valueLableQuery = QueryBuilders.termQuery(ESConstant.VALUELABEL+".keyword",queryRequest.getValueLabel()); + if (!("").equals(queryRequest.getValueLabel()) && null != queryRequest.getValueLabel()) { + QueryBuilder valueLableQuery = QueryBuilders.termQuery(ESConstant.VALUELABEL + ".keyword", queryRequest.getValueLabel()); boolQuery.must(valueLableQuery); } // 分类标签 // 因为是多选，因此需要判断，如果是多个的话，需要先用, 分隔，然后再用模糊匹配进行查询 - if(!("").equals(queryRequest.getCategoryLabel()) && null != queryRequest.getCategoryLabel()){ - // QueryBuilder categoryLableQuery = QueryBuilders.queryStringQuery(queryRequest.getCategoryLabel()).defaultField(ESConstant.CATEGORYLABEL); //+".keyword" - if(queryRequest.getCategoryLabel().contains(",")){ // 有多个标签查询，用should - String labels []= queryRequest.getCategoryLabel().split(","); + if (!("").equals(queryRequest.getCategoryLabel()) && null != queryRequest.getCategoryLabel()) { + // QueryBuilder categoryLableQuery = QueryBuilders.queryStringQuery(queryRequest.getCategoryLabel()).defaultField(ESConstant.CATEGORYLABEL); //+".keyword" + if (queryRequest.getCategoryLabel().contains(",")) { // 有多个标签查询，用should + String labels[] = queryRequest.getCategoryLabel().split(","); BoolQueryBuilder shouleLaelQuery = QueryBuilders.boolQuery(); - for (String label:labels) { - QueryBuilder categoryLableQuery = QueryBuilders.matchPhraseQuery(ESConstant.CATEGORYLABEL,label); + for (String label : labels) { + QueryBuilder categoryLableQuery = QueryBuilders.matchPhraseQuery(ESConstant.CATEGORYLABEL, label); shouleLaelQuery.should(categoryLableQuery); boolQuery.must(shouleLaelQuery); } - }else{ - QueryBuilder categoryLableQuery = QueryBuilders.matchPhraseQuery(ESConstant.CATEGORYLABEL,queryRequest.getCategoryLabel()); + } else { + QueryBuilder categoryLableQuery = QueryBuilders.matchPhraseQuery(ESConstant.CATEGORYLABEL, queryRequest.getCategoryLabel()); boolQuery.must(categoryLableQuery); } } @@ -64,7 +70,7 @@ public class TopicQueryService { } else { List areaList = siteRepository.findCidsByArea(queryRequest.getSearchArea()); List lowCaseAreaList = areaList.stream().map(String::toLowerCase).collect(Collectors.toList()); - boolQuery.must(QueryBuilders.termsQuery(ESConstant.EN_SOURCE+".keyword",lowCaseAreaList)); + boolQuery.must(QueryBuilders.termsQuery(ESConstant.EN_SOURCE, lowCaseAreaList)); // String searchArea = getSearchArea(queryRequest.getSearchArea()); // boolQuery.must(QueryBuilders.termQuery(ESConstant.AREA, searchArea)); } @@ -79,9 +85,9 @@ public class TopicQueryService { // cid 站点 String cid = queryRequest.getCid(); // taskId 任务ID - if(null == queryRequest.getTaskId() || ("").equals(queryRequest.getTaskId()) ){ + if (null == queryRequest.getTaskId() || ("").equals(queryRequest.getTaskId())) { logger.info("[TopicQueryService] queryByConditions_v1 没有任务ID，查询专题下全部任务"); - }else{ + } else { String taskId = queryRequest.getTaskId(); // 如果有任务ID 的话，查这个任务ID 对应的 crawlDataFlag 及时间范围，然后写到查询条件中 Task task = taskRepository.findAllBydel0AndTaskId(taskId); @@ -95,11 +101,11 @@ public class TopicQueryService { if (null == cid || ("").equals(cid) || ("test").equals(cid)) { logger.info("[TopicQueryService] queryByConditions_v1 查询全部站点"); } else { - if(cid.equals("Tmall") || cid.equals("Taobao")){ - boolQuery.must(QueryBuilders.termsQuery(ESConstant.EN_SOURCE+".keyword", "tmall","taobao")); - // boolQuery.should(QueryBuilders.termQuery(ESConstant.EN_SOURCE+".keyword", "taobao")); - }else { - boolQuery.must(QueryBuilders.termQuery(ESConstant.EN_SOURCE+".keyword", cid.toLowerCase())); + if (cid.equals("Tmall") || cid.equals("Taobao")) { + boolQuery.must(QueryBuilders.termsQuery(ESConstant.EN_SOURCE, "tmall", "taobao")); + // boolQuery.should(QueryBuilders.termQuery(ESConstant.EN_SOURCE, "taobao")); + } else { + boolQuery.must(QueryBuilders.termQuery(ESConstant.EN_SOURCE, cid.toLowerCase())); } } @@ -112,7 +118,7 @@ public class TopicQueryService { } Integer searchType = queryRequest.getSearchType(); - if(searchType == 2){ + if (searchType == 2) { return boolQuery; } // pubtime && crawlTime 发表时间，抓取时间 @@ -121,21 +127,23 @@ public class TopicQueryService { long crawlEndTime = queryRequest.getCrawlEndTime(); boolQuery.must(QueryBuilders.rangeQuery(ESConstant.CRAWLTIME).gte(crawlStartTime).lt(crawlEndTime)); } else { - long startTime =0L; - if(null != queryRequest.getStartTime() ) { + long startTime = 0L; + if (null != queryRequest.getStartTime()) { startTime = queryRequest.getStartTime(); } long endTime = 0L; - if(null != queryRequest.getEndTime()) { + if (null != queryRequest.getEndTime()) { endTime = queryRequest.getEndTime(); } - if(startTime == 0 && endTime == 0){ + if (startTime == 0 && endTime == 0) { logger.info("没有时间筛选，查询全部专题下的数据！"); - }else if(startTime == 0){ // 如果开始时间为空，那就查询小于 endTime 的所有数据 + // 如果开始时间为空，那就查询小于 endTime 的所有数据 + } else if (startTime == 0) { boolQuery.must(QueryBuilders.rangeQuery(ESConstant.PUBTIME).lt(endTime)); - }else if (endTime == 0){ // 如果结束时间为空，加就查大于 startTime 的所有数据 + // 如果结束时间为空，加就查大于 startTime 的所有数据 + } else if (endTime == 0) { boolQuery.must(QueryBuilders.rangeQuery(ESConstant.PUBTIME).gte(startTime)); - }else{ + } else { boolQuery.must(QueryBuilders.rangeQuery(ESConstant.PUBTIME).gte(startTime).lt(endTime)); } } @@ -148,49 +156,132 @@ public class TopicQueryService { Integer imageDownload = queryRequest.getWithImage(); Integer videoDownload = queryRequest.getWithVideo(); - if (null != withAppendix && withAppendix == 0) { // 选中无附件的，因此 is_download = false + // 4个全=0，全选， 没有查询条件 查全部 +// BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); +// if (withAppendix == 0 && fileDownload == 0 && imageDownload == 0 && videoDownload == 0) { +// // 不组装语句 +// } else if (withAppendix == 1) { +// // 查有附件的,有文件 或 有图片 或 有视频 +// if (fileDownload == 0 && imageDownload == 0 && videoDownload == 0) { +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); +// // 有图片或者有视频 +// }else if (fileDownload == 1 && imageDownload == 0 && videoDownload == 0){ +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); +// // 有文件或者有视频 +// }else if (fileDownload == 0 && imageDownload == 1 && videoDownload == 0){ +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); +// // 有图片或者有文件 +// }else if (fileDownload == 0 && imageDownload == 0 && videoDownload == 1){ +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); +// // 有视频 +// }else if (fileDownload == 1 && imageDownload == 1 && videoDownload == 0){ +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); +// // 有图片 +// }else if (fileDownload == 1 && imageDownload == 0 && videoDownload == 1){ +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); +// // 有文件 +// }else if (fileDownload == 0 && imageDownload == 1 && videoDownload == 1){ +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); +// } +// } else if (withAppendix == 0) { +// // 只选择 无附件 +// if (fileDownload == 1 && imageDownload == 1 && videoDownload == 1) { +// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.HAS_FILE, 0)); +// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 0)); +// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 0)); +// // 无附件 & 有视频 +// }else if(fileDownload == 1 && imageDownload == 1 && videoDownload == 0){ +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 0)); +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 0)); +// // 无附件 & 有图片 +// }else if(fileDownload == 1 && imageDownload == 0 && videoDownload == 1){ +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 0)); +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 0)); +// //无附件 & 有文件 +// }else if(fileDownload == 0 && imageDownload == 1 && videoDownload == 1){ +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); +// //无附件 & 有文件 无图片，无视频 后面三个不对啊！！！！ +// }else if(fileDownload == 1 && imageDownload == 0 && videoDownload == 0){ +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); +// //无附件 & 有文件 +// }else if(fileDownload == 0 && imageDownload == 1 && videoDownload == 0){ +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 0)); +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 0)); +// //无附件 & 有文件 +// }else if(fileDownload == 0 && imageDownload == 0 && videoDownload == 1){ +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 0)); +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); +// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 0)); +// } +// } + // 无附件=0，其他三个为1 fileDownload=1；imageDownload=1；videoDownload=1 + // 无附件=0，有图片 imageDownload=0；fileDownload=1；videoDownload=1 + // 无附件=0，有视频 videoDownload=0；fileDownload=1；imageDownload=1 + // 无附件=0，有文件 fileDownload=0；imageDownload=1；videoDownload=1 + // 无附件=1 有图片，有视频，有附件 fileDownload=0；imageDownload=0；videoDownload=0 + // 无附件=1 fileDownload=0；imageDownload=0；videoDownload=0 +// 选中无附件的，因此 is_download = false ==1 表示未选择 + // withAppendix == 1 表示选中了 "有附件，有图片，有视频" 没有选"无附件"，所以 isDownload = true + if (null != withAppendix && withAppendix == 1) { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.IS_DOWNLOAD, false)); +// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)) +// .should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)) +// .should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); +// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.IS_DOWNLOAD, true)); if (null != fileDownload && fileDownload == 0) { - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); } if (null != imageDownload && imageDownload == 0) { - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); } if (null != videoDownload && videoDownload == 0) { - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); } boolQuery.must(boolQueryBuilder); - } else { // 当选中有附件的 - boolQuery.must(QueryBuilders.termQuery(ESConstant.IS_DOWNLOAD, true)); + } else { // 当选中无附件的 withAppendix == 0 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - if (null != fileDownload && fileDownload == 1 && null != imageDownload && imageDownload == 1 && null != videoDownload && videoDownload == 1) { - boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1)); - boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1)); - boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.IS_DOWNLOAD, false)); + if (null != fileDownload && fileDownload == 1 + && null != imageDownload && imageDownload == 1 + && null != videoDownload && videoDownload == 1) { + boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); + boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); + boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); } else if (null != fileDownload && fileDownload == 0 && null != imageDownload && imageDownload == 1 && null != videoDownload && videoDownload == 1) { - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); } else if (null != fileDownload && fileDownload == 1 && null != imageDownload && imageDownload == 0 && null != videoDownload && videoDownload == 1) { - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); } else if (null != fileDownload && fileDownload == 1 && null != imageDownload && imageDownload == 1 && null != videoDownload && videoDownload == 0) { - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); } else if (null != fileDownload && fileDownload == 0 && null != imageDownload && imageDownload == 0 && null != videoDownload && videoDownload == 1) { - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1)); - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); } else if (null != imageDownload && imageDownload == 0 && null != videoDownload && videoDownload == 0 && null != fileDownload && fileDownload == 1) { - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1)); - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); } else if (null != videoDownload && videoDownload == 0 && null != fileDownload && fileDownload == 0 && null != imageDownload && imageDownload == 1) { - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1)); - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1)); - } else if (null != videoDownload && videoDownload == 0 && null != fileDownload && fileDownload == 0 && null != imageDownload && imageDownload == 0) { - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1)); - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1)); - boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); + } else if (null != videoDownload && videoDownload == 0 + && null != fileDownload && fileDownload == 0 + && null != imageDownload && imageDownload == 0) { + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1)); + boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1)); } boolQuery.must(boolQueryBuilder); } - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } return boolQuery; @@ -206,10 +297,10 @@ public class TopicQueryService { // primary = 1 boolQuery.must(QueryBuilders.termQuery(ESConstant.PRIMARY, 1)); // authorId = - if(null != queryRequest.getAuthorId() && !("").equals(queryRequest.getAuthorId())){ + if (null != queryRequest.getAuthorId() && !("").equals(queryRequest.getAuthorId())) { boolQuery.must(QueryBuilders.termQuery(ESConstant.AUTHORID, queryRequest.getAuthorId())); } - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } return boolQuery; @@ -225,22 +316,21 @@ public class TopicQueryService { // primary = 2 boolQuery.must(QueryBuilders.termQuery(ESConstant.PRIMARY, 2)); // authorId = - if(null != queryRequest.getAuthorId() && !("").equals(queryRequest.getAuthorId())){ + if (null != queryRequest.getAuthorId() && !("").equals(queryRequest.getAuthorId())) { boolQuery.must(QueryBuilders.termQuery(ESConstant.AUTHORID, queryRequest.getAuthorId())); } - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } return boolQuery; } - /** * 根据 searchScope 中的查询条件组装查询语句 */ - public BoolQueryBuilder buildSearchTextBuilder(String searchScope,String keyword) { - return ParseSearchScopeService.getSearchScopeQuery(searchScope,keyword); + public BoolQueryBuilder buildSearchTextBuilder(String searchScope, String keyword) { + return ParseSearchScopeService.getSearchScopeQuery(searchScope, keyword); } /** @@ -268,8 +358,8 @@ public class TopicQueryService { * 组装渠道查询语句 */ private QueryBuilder buildChannelQueryBuilder(QueryRequest queryRequest) { - String [] channel = queryRequest.getSiteTypes().split(","); - List channelList =getChannelList(channel); + String[] channel = queryRequest.getSiteTypes().split(","); + List channelList = getChannelList(channel); if (TObjectUtils.isListEmpty(channelList)) { return null; } @@ -279,18 +369,26 @@ public class TopicQueryService { /** * 查询渠道 */ - private List getChannelList(String [] channel){ + private List getChannelList(String[] channel) { List channels = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("0","社交媒体"); - map.put("1","新闻资讯"); - map.put("2","博客智库"); - map.put("3","论坛贴吧"); - map.put("4","网络视频"); - map.put("5","电商网站"); - map.put("6","搜索引擎"); - map.put("7","生活方式"); - for (String id:channel) { + Map map = new HashMap<>(); +// map.put("0","社交媒体"); +// map.put("1","新闻资讯"); +// map.put("2","博客智库"); +// map.put("3","论坛贴吧"); +// map.put("4","网络视频"); +// map.put("5","电商网站"); +// map.put("6","搜索引擎"); +// map.put("7","生活方式"); + map.put("0", "社交媒体"); + map.put("1", "政企官网"); + map.put("2", "智库资讯"); + map.put("3", "新闻媒体"); + map.put("4", "网络视频"); + map.put("5", "电商网站"); + map.put("6", "搜索引擎"); + map.put("7", "内容分享"); + for (String id : channel) { channels.add(map.get(id)); } return channels; diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/common/CrudService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/common/CrudService.java index 76b95f5..bde7d03 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/common/CrudService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/common/CrudService.java @@ -1,12 +1,10 @@ package com.bfd.mf.common.service.common; -import com.alibaba.fastjson.JSONObject; import com.bfd.mf.common.web.entity.mysql.Model; -import com.bfd.mf.common.web.vo.params.QueryRequest; import org.springframework.data.repository.CrudRepository; import java.io.Serializable; -import java.util.*; +import java.util.Optional; public abstract class CrudService> implements Serializable { diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ClusterService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ClusterService.java index 116136f..f9d2e48 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ClusterService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ClusterService.java @@ -24,7 +24,9 @@ public class ClusterService extends CrudService { * 只对应获取 mini && normal cluster */ public Cluster findClusterByType(int type) { - if (type == Cluster.CLUSTER_TYPE.mini_cluster_type || type == Cluster.CLUSTER_TYPE.normal_cluster_type) { + if (type == Cluster.CLUSTER_TYPE.mini_cluster_type + || type == Cluster.CLUSTER_TYPE.normal_cluster_type + || type == Cluster.CLUSTER_TYPE.special_cluster_type) { return repo.findClusterByType(type); } return null; diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsCommonService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsCommonService.java index b23d9ed..2884432 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsCommonService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsCommonService.java @@ -1,15 +1,16 @@ package com.bfd.mf.common.service.es; -import com.bfd.mf.common.util.constants.ESConstant; import com.bfd.mf.common.util.ESServerUtils; +import com.bfd.mf.common.util.constants.ESConstant; import com.bfd.nlp.common.util.string.TStringUtils; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.*; +import java.util.List; @Service public class EsCommonService { diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorCountService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorCountService.java index c328c99..3ba5c3a 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorCountService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorCountService.java @@ -8,7 +8,6 @@ import com.bfd.mf.common.web.vo.params.QueryRequest; import com.bfd.mf.config.BFDApiConfig; import com.bfd.mf.service.SearchAuthorService; import com.bfd.nlp.common.util.string.TStringUtils; -import org.apache.commons.collections.map.HashedMap; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.slf4j.Logger; @@ -17,7 +16,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; -import java.util.*; @Service public class EsQueryAuthorCountService { diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorService.java index 186f27a..1ea0e7a 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorService.java @@ -9,14 +9,20 @@ import com.bfd.mf.common.web.vo.params.QueryRequest; import com.bfd.mf.config.BFDApiConfig; import com.bfd.mf.service.SearchAuthorService; import com.bfd.nlp.common.util.string.TStringUtils; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchPhraseQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; @Service public class EsQueryAuthorService { @@ -68,6 +74,7 @@ public class EsQueryAuthorService { siteIconMap.put(map.get("cid").toString().toLowerCase(), map.get("site_icon").toString()); } } + List newResult = new ArrayList<>(); for (JSONObject json: result) { JSONObject newJson= json; @@ -96,7 +103,8 @@ public class EsQueryAuthorService { BoolQueryBuilder boolQueryBuilder = null; boolQueryBuilder = topicQueryService.queryByConditions_v1(queryRequest); // 二次查询： 关键词不为空就添加关键词查询语句 = 0 content 1 title 2 author 3 con+tit 4con+aut 5con+com - Integer searchType = queryRequest.getSearchType(); // 单选 0:主贴;1:评论;2:用户 + // 单选 0:主贴;1:评论;2:用户 + Integer searchType = queryRequest.getSearchType(); // String searchScope = queryRequest.getSearchScope(); //复选 0:标题;1:正文;2:作者 多个用,分割，例 “0,1” String keyword = queryRequest.getKeyword(); @@ -104,17 +112,20 @@ public class EsQueryAuthorService { boolQueryBuilder.filter(searchTextBuilder); // Map fields = new HashedMap(); if (TStringUtils.isNotEmpty(keyword)) { - if(searchType == 0){ // 主贴的话 查 标题和内容 + // 主贴的话 查 标题和内容 + if(searchType == 0){ MatchPhraseQueryBuilder titleQuery = QueryBuilders.matchPhraseQuery(ESConstant.TITLE, keyword).slop(0); MatchPhraseQueryBuilder contentQuery = QueryBuilders.matchPhraseQuery(ESConstant.CONTENT, keyword).slop(0); QueryBuilder queryBuilder = QueryBuilders.boolQuery().should(titleQuery).should(contentQuery); bqb.must(queryBuilder); - }else if (searchType == 1){ // 评论的话 查 评论内容 + // 评论的话 查 评论内容 + }else if (searchType == 1){ // MatchPhraseQueryBuilder contentQuery = QueryBuilders.matchPhraseQuery(ESConstant.CONTENT, keyword).slop(0); // QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(contentQuery); // qb.must(queryBuilder); boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(ESConstant.CONTENT, keyword).slop(0)); - }else if (searchType == 2){ // 用户 就只查 用户名 + // 用户 就只查 用户名 + }else if (searchType == 2){ boolQueryBuilder.must(QueryBuilders.queryStringQuery("*"+keyword+"*").field(ESConstant.AUTHOR)); } } diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQMini.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQMini.java index 1ae48ea..e94837d 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQMini.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQMini.java @@ -11,7 +11,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; -import java.util.*; +import java.util.ArrayList; +import java.util.List; @Service public class EsQueryServiceForSQMini { @@ -28,7 +29,8 @@ public class EsQueryServiceForSQMini { // 注册数据查询来源 clusterName = bfdApiConfig.esMiniName(); String sourceAddress [] = bfdApiConfig.esMiniAddress(); - EsUtils.registerCluster(clusterName, sourceAddress);// 配置文件中的 es-source + // 配置文件中的 es-source + EsUtils.registerCluster(clusterName, sourceAddress); } @@ -38,11 +40,16 @@ public class EsQueryServiceForSQMini { BoolQueryBuilder boolQueryBuilder = getQueryBuilder.getQueryBuilder(queryRequest); //logger.info("[EsQueryService] queryDataFromOneSubject: indexName = " + indexName[0] + "; qb:\n" + "{}.", boolQueryBuilder.toString()); logger.info("[EsQueryService] queryDataFromOneSubject: indexName : {}" ,indexName[0]); - Integer limit = queryRequest.getLimit(); //每页的数量 - Integer start = (queryRequest.getPage() - 1) * limit; //起始页(0,20,40....) - String orderFlag = queryRequest.getOrder(); // 排序方式 asc/desc - String sortFlag = queryRequest.getSidx(); // 排序字段 - Integer searchType = queryRequest.getSearchType(); // 主贴、评论、还是用户？ + //每页的数量 + Integer limit = queryRequest.getLimit(); + //起始页(0,20,40....) + Integer start = (queryRequest.getPage() - 1) * limit; + // 排序方式 asc/desc + String orderFlag = queryRequest.getOrder(); + // 排序字段 + String sortFlag = queryRequest.getSidx(); + // 主贴、评论、还是用户？ + Integer searchType = queryRequest.getSearchType(); List result = EsUtils.query(clusterName, indexName, boolQueryBuilder, sortFlag, orderFlag, limit, start, searchType); return result; }catch (Exception e){ @@ -50,26 +57,39 @@ public class EsQueryServiceForSQMini { } } + /** + * 查询数据量 + * @param indexName + * @param queryRequest + * @return + */ public Long queryDataCountFromOneSubject(String[] indexName, QueryRequest queryRequest) { try { logger.debug("[EsQueryServiceForSQMini - 专题] queryDataCountFromOneSubject ..."); BoolQueryBuilder boolQueryBuilder = getQueryBuilder.getQueryBuilder(queryRequest); Integer searchType = queryRequest.getSearchType(); Integer size = queryRequest.getLimit(); - //logger.info("[EsQueryService] queryDataCountFromOneSubject indexName = " + indexName + "; qb:" + "{}.", boolQueryBuilder.toString()); - Long totalCount = EsUtils.queryTotalCount(clusterName, indexName, boolQueryBuilder, searchType ); + Long totalCount = EsUtils.queryTotalCountNew(clusterName, indexName, boolQueryBuilder, searchType ); + //System.out.println("EsQueryServiceForSQMini : queryDataCountFromOneSubject " + totalCount); return totalCount; }catch (Exception e){ return 0L; } } + /** + * 复制示例专题的数据到新的专题 + * @param indexName + * @param queryRequest + * @return + */ public JSONObject exportDataFromOneSubject(String[] indexName, QueryRequest queryRequest) { try { logger.info("[EsQueryServiceForSQMini - 专题] exportDataFromOneSubject start ..."); BoolQueryBuilder boolQueryBuilder =getQueryBuilder.getQueryBuilder(queryRequest); // logger.info("[EsQueryServiceForSQMini-专题] queryDataFromOneSubject: indexName = " + indexName + "; qb:" + "{}.", boolQueryBuilder.toString()); - Integer limit = queryRequest.getLimit(); //每页的数量 + //每页的数量 + Integer limit = queryRequest.getLimit(); String scrollId = queryRequest.getScrollId(); Integer searchType = queryRequest.getSearchType(); JSONObject result = EsUtils.queryForExport(clusterName, indexName, boolQueryBuilder, limit,scrollId,searchType); @@ -80,6 +100,41 @@ public class EsQueryServiceForSQMini { } } + public long reIndexData(String indexList, String newIndex) { + try{ + long created = EsUtils.reIndex(clusterName,indexList,newIndex); + return created; + }catch (Exception e){ + e.printStackTrace(); + return 0; + } + } + + public void deleteBySubjectId(String indexName) { + try{ + EsUtils.delIndex(clusterName,indexName); + }catch (Exception e){ + e.printStackTrace(); + } + } + + public void deleteBySubjectIdByCid(String indexName, String cid) { + try{ + EsUtils.delIndexByCid(clusterName,indexName,cid); + }catch (Exception e){ + e.printStackTrace(); + } + + } + + public void deleteBySubjectIdByCrawlDataFlag(String indexName, String cid, String crawlDataFlag) { + try{ + EsUtils.delIndexByCrawlDataFlag(clusterName,indexName,cid,crawlDataFlag); + }catch (Exception e){ + e.printStackTrace(); + } + } + // public JSONObject exportDataFromOneSubjectTestGroupBy(String[] indexName, QueryRequest queryRequest) { // try { // logger.info("[exportDataFromOneSubjectTestGroupBy - 专题] exportDataFromOneSubject start ..."); diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQNormal.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQNormal.java index d122342..6594e24 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQNormal.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQNormal.java @@ -11,7 +11,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; -import java.util.*; +import java.util.List; @Service public class EsQueryServiceForSQNormal { diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/GetQueryBuilder.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/GetQueryBuilder.java index 350132d..9cbc653 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/GetQueryBuilder.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/GetQueryBuilder.java @@ -4,8 +4,11 @@ import com.bfd.mf.common.service.cache.TopicQueryService; import com.bfd.mf.common.util.constants.ESConstant; import com.bfd.mf.common.web.vo.params.QueryRequest; import com.bfd.nlp.common.util.string.TStringUtils; -import org.apache.commons.collections.map.HashedMap; -import org.elasticsearch.index.query.*; +import org.apache.commons.collections4.map.HashedMap; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchPhraseQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; @@ -32,10 +35,12 @@ public class GetQueryBuilder { // 如果要根据ID 查询数据 如果查ID 的，后面的条件就不用查了。 if(null != queryRequest.getDataIds() && !("").equals(queryRequest.getDataIds())) { + String taskId = queryRequest.getTaskId(); String dataIds = queryRequest.getDataIds(); List dataIdList = getDataIdList(dataIds); QueryBuilder queryBuilder = QueryBuilders.termsQuery(ESConstant.DATA_ID, dataIdList); - boolQueryBuilder = QueryBuilders.boolQuery().filter(queryBuilder); + boolQueryBuilder = QueryBuilders.boolQuery().filter(queryBuilder) + .must(QueryBuilders.termQuery("taskId",taskId)); qb.must(boolQueryBuilder); return qb; } @@ -60,7 +65,6 @@ public class GetQueryBuilder { boolQueryBuilder.must(QueryBuilders.queryStringQuery("*"+keyword+"*").field(ESConstant.AUTHOR)); } } - qb.must(boolQueryBuilder); return qb; } diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java index e0939a0..20a92b6 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java @@ -64,7 +64,7 @@ public class ParseSearchScopeService { .must(QueryBuilders.termQuery(ESConstant.DOC_TYPE,ESConstant.ITEM))); } else if(searchType == 2){ searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY, 2)) - .must(QueryBuilders.termsQuery(ESConstant.PAGETYPE,"userInfoPage","newsuser")); + .must(QueryBuilders.termsQuery(ESConstant.PAGETYPE,"userInfoPage","newsuser","videoAccount")); } return searchScopeQuery; } diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java index 583530b..3f38359 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java @@ -12,7 +12,10 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchPhraseQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/text/TextService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/text/TextService.java index 2f90577..cf5c3c9 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/text/TextService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/text/TextService.java @@ -2,7 +2,6 @@ package com.bfd.mf.common.service.text; import com.alibaba.fastjson.JSONObject; import com.bfd.mf.common.util.constants.ESConstant; -import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity; import com.bfd.mf.config.BFDApiConfig; import net.logstash.logback.encoder.org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; @@ -27,7 +26,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java index a7890e1..a2b6fb2 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java @@ -1,11 +1,10 @@ package com.bfd.mf.common.util.analysis; import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; import com.bfd.mf.common.util.constants.ESConstant; import com.bfd.mf.common.util.enums.SpreadEnums; import com.bfd.mf.common.util.utility.DateUtil; -import com.bfd.mf.common.web.vo.view.analysis.*; +import com.bfd.mf.common.web.vo.view.analysis.DataCount; import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity; import com.bfd.nlp.common.util.object.TObjectUtils; import org.slf4j.Logger; @@ -14,7 +13,6 @@ import org.slf4j.LoggerFactory; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; -import java.util.stream.IntStream; public class DataAnalysisUtils { diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java index ab9588e..3c23588 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java @@ -10,7 +10,6 @@ import org.slf4j.LoggerFactory; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; public class ResponseParseUtils { private static Logger logger = LoggerFactory.getLogger(ResponseParseUtils.class); diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java index 19f23aa..5647efe 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java @@ -265,11 +265,13 @@ public class ESConstant { /** * SQ 4.0新增 UGC EGC PGC */ - public static final String SEARCH_SCOPE_UGC = "ugc"; +// public static final String SEARCH_SCOPE_UGC = "ugc"; +// public static final String SEARCH_SCOPE_EGC = "egc"; +// public static final String SEARCH_SCOPE_PGC = "pgc"; - public static final String SEARCH_SCOPE_EGC = "egc"; - - public static final String SEARCH_SCOPE_PGC = "pgc"; + public static final String HAS_IMAGE = "hasImage"; + public static final String HAS_VIDEO = "hasVideo"; + public static final String HAS_FILE = "hasFile"; /** * 关键词 */ @@ -365,6 +367,7 @@ public class ESConstant { public static String CONTENT_SIMHASH = "contentSimHash"; public static String QUOTE_COUNT = "quoteCount"; public static String COLLE_CTCOUNT = "collectCount"; + public static String VIEW_CNT = "viewCnt"; // 左侧检索内容那块要显示的数字 public static String CONTENTCOUNT = "contentCount"; public static String COMMENTCOUNT = "commentCount"; @@ -775,7 +778,7 @@ public class ESConstant { ESConstant.PUBTIME , // 发表时间 ESConstant.QUOTE_COUNT , // 转发数 -// ESConstant.COMMENTS_COUNT , // 评论数 + ESConstant.COMMENTS_COUNT , // 评论数 ESConstant.ATTITUDES_COUNT , // 点赞数 ESConstant.PRICE, // 价格 ESConstant.POST_COUNT, // 销量 diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java index df17972..53a46d7 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java @@ -3,6 +3,10 @@ package com.bfd.mf.common.util.es; import com.alibaba.fastjson.JSONObject; import com.bfd.mf.common.util.constants.ESConstant; import com.google.common.collect.Maps; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -20,11 +24,16 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.*; import org.elasticsearch.script.Script; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality; +import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.slf4j.Logger; @@ -32,9 +41,10 @@ import org.slf4j.LoggerFactory; import org.springframework.util.Assert; import java.net.InetAddress; -import java.util.*; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public abstract class EsUtils { @@ -75,24 +85,62 @@ public abstract class EsUtils { Integer size, Integer from, Integer searchType) { TransportClient client = getClient(clusterName); - boolean options = true; boolean optionsf = false; - -// CollapseBuilder collapseBuilder = new CollapseBuilder(ESConstant.DATA_ID + ".keyword"); -// if(searchType == 0 || searchType == 2) { // 表示查的是主贴，电商数据要根据 docId 消重才行。 -// collapseBuilder = new CollapseBuilder(ESConstant.DOC_ID); -// } - // from +size 的 分页 查询方式 + // 现在不同任务的同一条数据不做消重，因此同一个DOCID 的数据会有多条。因此只有查主贴的时候需要用DOCID 消重 + CollapseBuilder collapseBuilder = new CollapseBuilder(ESConstant.DATA_ID); + if (searchType == 0) { + collapseBuilder = new CollapseBuilder(ESConstant.DOC_ID); + } + // from + size 的 分页 查询方式 SearchRequestBuilder requestBuilder = client.prepareSearch().setIndices(index) .setIndicesOptions(IndicesOptions.fromOptions(options, options, options, optionsf)) .addSort(sortFlag, orderFlag.equals(ESConstant.ASC) ? SortOrder.ASC : SortOrder.DESC) .setQuery(queryBuilder) - // .setCollapse(collapseBuilder) + .setCollapse(collapseBuilder) .setSize(size) .setFrom(from); + System.out.println(requestBuilder); + SearchResponse searchResponse = requestBuilder.execute().actionGet(); + // List dataList = Lists.newLinkedList(); + List dataList = new ArrayList<>(); + if (searchResponse.getHits().totalHits > 0) { + SearchHit[] hits = searchResponse.getHits().getHits(); + for (int i = 0; i < hits.length; i++) { + JSONObject data = new JSONObject(); + data.putAll(hits[i].getSourceAsMap()); + data.put("subjectId", hits[i].getIndex() + .replace("cl_major_", "") + .replace("cl_subject_", "") + .replace("cl_special_1.0_", "")); + dataList.add(data); + } + } + return dataList; + } + public static List queryNew(String clusterName, String[] index, + final QueryBuilder queryBuilder, + String sortFlag, String orderFlag, + Integer size, Integer from, + Integer searchType) { + TransportClient client = getClient(clusterName); + boolean options = true; + boolean optionsf = false; + // 现在不同任务的同一条数据不做消重，因此同一个DOCID 的数据会有多条。因此只有查主贴的时候需要用DOCID 消重 + CollapseBuilder collapseBuilder = new CollapseBuilder(ESConstant.DATA_ID); + if (searchType == 0) { + collapseBuilder = new CollapseBuilder(ESConstant.DOC_ID); + } + // from + size 的 分页 查询方式 + SearchRequestBuilder requestBuilder = client.prepareSearch().setIndices(index) + .setIndicesOptions(IndicesOptions.fromOptions(options, options, options, optionsf)) + .addSort(sortFlag, orderFlag.equals(ESConstant.ASC) ? SortOrder.ASC : SortOrder.DESC) + .setQuery(queryBuilder) + .setCollapse(collapseBuilder) + .setSize(size) + .setFrom(from); System.out.println(requestBuilder); SearchResponse searchResponse = requestBuilder.execute().actionGet(); @@ -103,29 +151,41 @@ public abstract class EsUtils { for (int i = 0; i < hits.length; i++) { JSONObject data = new JSONObject(); data.putAll(hits[i].getSourceAsMap()); - data.put("subjectId",hits[i].getIndex().replace("cl_major_","").replace("cl_subject_","")); + data.put("subjectId", hits[i].getIndex() + .replace("cl_major_", "") + .replace("cl_subject_", "") + .replace("cl_special_1.0_", "")); dataList.add(data); } } return dataList; } + /** + * 查询数据量 count + * @param clusterName + * @param index + * @param queryBuilder + * @param searchType + * @return + */ public static Long queryTotalCount(String clusterName, String[] index, QueryBuilder queryBuilder, Integer searchType) { + TransportClient client = getClient(clusterName); + Map settings = new HashMap<>(); + settings.put("index.max_result_window", "200000000"); + updSetting(index[0], settings, client); boolean options = true; boolean optionsf = false; -// CollapseBuilder collapseBuilder = new CollapseBuilder(ESConstant.DATA_ID+".keyword"); -// if(searchType == 0) { // 表示查的是主贴，电商数据要根据 docId 消重才行。 -// collapseBuilder = new CollapseBuilder(ESConstant.DOC_ID); -// } - int querysize = 400000; + // 现在不通任务的同一条数据不做消重，因此同一个DOCID 的数据会有多条。因此只有查主贴的时候需要用 DOCID 消重 + CollapseBuilder collapseBuilder = new CollapseBuilder(ESConstant.DATA_ID); + if (searchType == 0) { + collapseBuilder = new CollapseBuilder(ESConstant.DOC_ID); + } try { - TransportClient client = getClient(clusterName); + // JSONObject settings = JSONObject.parseObject("{\"max_result_window\": \"200000000\"}"); - Map settings = new HashMap<>(); - settings.put("index.max_result_window","200000000"); - updSetting(index[0],settings,client); SearchResponse response = null; // if(searchType == 0){ // 查的是主贴，需要用 limit 取数不然 默认limit =10 // response = client.prepareSearch().setIndices(index) @@ -138,14 +198,14 @@ public abstract class EsUtils { response = client.prepareSearch().setIndices(index) .setIndicesOptions(IndicesOptions.fromOptions(options, options, options, optionsf)) .setQuery(queryBuilder) -// .setCollapse(collapseBuilder) + .setCollapse(collapseBuilder) + .setSize(1000) .execute().actionGet(); // } - - Long size = response.getHits().getTotalHits(); - logger.info("[queryTotalCount] 聚合前的总量: {} 这个是聚合前的数据量 ",size); + Long size = response.getHits().getTotalHits(); + logger.info("[queryTotalCount] 聚合前的总量: {} 这个是聚合前的数据量 ", size); Long aggrSize = Long.valueOf(response.getHits().getHits().length); - logger.info("[queryTotalCount] 聚合后的数据量: {} ",aggrSize); + logger.info("[queryTotalCount] 聚合后的数据量: {} ", aggrSize); // 当聚合后的结果为 10000 时 ，或者 是评论数据的时候，就不用聚合了 // if(aggrSize == 10000 || searchType == 1 || searchType == 2){ // size = size; @@ -153,21 +213,49 @@ public abstract class EsUtils { // size = aggrSize; // } return size; - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } return 0L; } + public static Long queryTotalCountNew(String clusterName, String[] index, + QueryBuilder queryBuilder, + Integer searchType) { + + TransportClient client = getClient(clusterName); + boolean options = true; + boolean optionsf = false; + // 现在不同任务的同一条数据不做消重，因此同一个DOCID 的数据会有多条。因此只有查主贴的时候需要用DOCID 消重 + String count = "count"; + AggregationBuilder aggregation ; + if(searchType == 0) { + aggregation = AggregationBuilders.cardinality(count).field(ESConstant.DOC_ID); + }else{ + aggregation = AggregationBuilders.cardinality(count).field(ESConstant.DATA_ID); + } + //searchSourceBuilder.aggregation(aggregation); + // from + size 的 分页 查询方式 + SearchRequestBuilder requestBuilder = client.prepareSearch().setIndices(index) + .setIndicesOptions(IndicesOptions.fromOptions(options, options, options, optionsf)) + .setQuery(queryBuilder) + .addAggregation(aggregation); + + // System.out.println(requestBuilder); + Aggregations aggregations = requestBuilder.get().getAggregations(); + Cardinality cardinality = aggregations.get(count); + // System.out.println("1111 : "+cardinality.getValue()); + return cardinality.getValue(); + } public static JSONObject queryForExport(String clusterName, - String indexName [], + String indexName[], BoolQueryBuilder boolQueryBuilder, Integer limit, String scrollId, Integer searchType) { - Map result = new HashMap<>(); + Map result = new HashMap<>(); TransportClient client = getClient(clusterName); SearchResponse searchResponse = null; try { @@ -181,9 +269,9 @@ public abstract class EsUtils { .setScroll(new TimeValue(300000)) .execute() .actionGet();//注意:首次搜索并不包含数据 - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); - logger.error(" queryForExport 首次 查询报错！！！IndexName : {} " , indexName[0]); + logger.error(" queryForExport 首次 查询报错！！！IndexName : {} ", indexName[0]); } } else { System.out.println("翻页查询"); @@ -200,20 +288,22 @@ public abstract class EsUtils { } //获取总数量 long totalCount = searchResponse.getHits().getTotalHits(); - int page=(int)totalCount/( 2 * limit);//计算总页数,每次搜索数量为分片数*设置的size大小 + //计算总页数,每次搜索数量为分片数*设置的size大小 + int page = (int) totalCount / (2 * limit); //int page = 2 * limit; - // System.out.println("queryForExport : " + totalCount + " ; page = " + page + " ; scrollId = " + searchResponse.getScrollId()); + // System.out.println("queryForExport : " + totalCount + " ; page = " + page + " ; scrollId = " + searchResponse.getScrollId()); for (int i = 0; i <= page; i++) { //再次发送请求,并使用上次搜索结果的ScrollId List monitorLists = parseSearchResponse(searchResponse); - if(monitorLists.size() == 0){ // 回收 scrollId - boolean clear = clearScroll(client,scrollId); + // 回收 scrollId + if (monitorLists.size() == 0) { + boolean clear = clearScroll(client, scrollId); } result.put(ESConstant.MONITORLISTS, monitorLists); } result.put(ESConstant.ALLDOCNUMBER, totalCount); result.put(ESConstant.SCROLLID, searchResponse.getScrollId()); - }catch (Exception e){ + } catch (Exception e) { logger.info("[EsUtils] queryForExport error !!! "); result.put(ESConstant.MONITORLISTS, new ArrayList()); result.put(ESConstant.ALLDOCNUMBER, 0L); @@ -225,7 +315,7 @@ public abstract class EsUtils { /** * 不在使用滚动时清除id */ - public static boolean clearScroll( TransportClient client ,String scrollId) { + public static boolean clearScroll(TransportClient client, String scrollId) { ClearScrollRequestBuilder clearScrollRequestBuilder = client.prepareClearScroll(); clearScrollRequestBuilder.addScrollId(scrollId); ClearScrollResponse response = clearScrollRequestBuilder.get(); @@ -240,7 +330,7 @@ public abstract class EsUtils { for (SearchHit searchHit : hits.getHits()) { try { JSONObject json = JSONObject.parseObject(searchHit.getSourceAsString()); - json.put("subjectId",searchHit.getIndex()); + json.put("subjectId", searchHit.getIndex()); result.add(json); // WriteMethod.writeMethod("dataIds",searchHit.getSourceAsMap().get("dataId").toString()); } catch (Exception e) { @@ -251,28 +341,28 @@ public abstract class EsUtils { } // 封装方法bai - public static boolean updSetting(String index, Map settings,TransportClient client) { + public static boolean updSetting(String index, Map settings, TransportClient client) { // 先判断索引是否存在 - boolean boo = isExistsIndex(index,client); - if(boo) { + boolean boo = isExistsIndex(index, client); + if (boo) { UpdateSettingsRequest request = Requests.updateSettingsRequest(index).settings(settings); UpdateSettingsResponse response = client.admin().indices().updateSettings(request).actionGet(); return response.isAcknowledged(); - }else{ + } else { return false; } } - private static boolean isExistsIndex(String indexName,TransportClient client){ - IndicesExistsResponse response = client.admin().indices().exists( + private static boolean isExistsIndex(String indexName, TransportClient client) { + IndicesExistsResponse response = client.admin().indices().exists( new IndicesExistsRequest().indices(indexName)).actionGet(); return response.isExists(); } - public static Terms queryByTag(String clusterName, String index,String filter, + public static Terms queryByTag(String clusterName, String index, String filter, QueryBuilder queryBuilder, - AggregationBuilder aggregationBuilder){ + AggregationBuilder aggregationBuilder) { TransportClient client = getClient(clusterName); SearchRequestBuilder requestBuilder = client.prepareSearch(index) @@ -280,9 +370,9 @@ public abstract class EsUtils { .setQuery(queryBuilder) .addAggregation(aggregationBuilder); - logger.info("requestBuilder: " , requestBuilder.toString().replace("\n","").replace("\r","").replace(" ","")); + logger.info("requestBuilder: ", requestBuilder.toString().replace("\n", "").replace("\r", "").replace(" ", "")); SearchResponse response = requestBuilder.execute().actionGet(); - Terms aggregation = response.getAggregations().get(filter+"Tag"); + Terms aggregation = response.getAggregations().get(filter + "Tag"); return aggregation; } @@ -298,18 +388,18 @@ public abstract class EsUtils { .actionGet();//注意:首次搜索并不包含数据 } catch (Exception e) { e.printStackTrace(); - logger.error(" queryForExport 首次 查询报错！！！IndexName : {} " ,indexName[0]); + logger.error(" queryForExport 首次 查询报错！！！IndexName : {} ", indexName[0]); } //再次发送请求,并使用上次搜索结果的ScrollId List monitorLists = parseSearchResponse(searchResponse); - if(monitorLists.size() >0){ + if (monitorLists.size() > 0) { return monitorLists.get(0); - }else{ + } else { return new JSONObject(); } } - public static long updateByDocId(String clusterName, String index,String script, + public static long updateByDocId(String clusterName, String index, String script, String docId, Map params) { TransportClient client = getClient(clusterName); try { @@ -323,9 +413,74 @@ public abstract class EsUtils { updateRequest.setRefreshPolicy("true"); UpdateResponse response = client.update(updateRequest).get(); return response.getVersion(); - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); return 0L; } } + + public static long reIndex(String clusterName, String indexList, String newIndex) { + try { + TransportClient client = getClient(clusterName); + System.out.println(indexList + " *** " + newIndex); + ReindexRequestBuilder builder = ReindexAction.INSTANCE + .newRequestBuilder(client) + .source(indexList) + .destination(newIndex); + String newAliex = newIndex.replace("cl_special_1.0_", "cl_major_"); + BulkByScrollResponse response = builder.get(); + + IndicesAliasesRequestBuilder indicesBuilder = IndicesAliasesAction.INSTANCE.newRequestBuilder(client).addAlias(newIndex, newAliex); + IndicesAliasesResponse IndicesResponse = indicesBuilder.get(); + System.out.println("******* : " + response); + System.out.println("##### : " + IndicesResponse); + return response.getCreated(); + } catch (Exception e) { + e.printStackTrace(); + return 0; + } + } + + public static void delIndex(String clusterName, String index) { + try { + TransportClient client = getClient(clusterName); + System.out.println("**** " + index); + DeleteIndexResponse response = client.admin().indices().prepareDelete(index).get(); + System.out.println(response.isAcknowledged()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void delIndexByCid(String clusterName, String indexName, String cid) { + try { + TransportClient client = getClient(clusterName); + System.out.println("**** " + indexName); + cid = cid.toLowerCase(); + DeleteByQueryRequestBuilder builder = DeleteByQueryAction.INSTANCE.newRequestBuilder(client) + .filter(QueryBuilders.matchQuery("enSource", cid)).source(indexName); + BulkByScrollResponse response2 = builder.get(); + long deleted = response2.getDeleted(); + System.out.println(deleted); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void delIndexByCrawlDataFlag(String clusterName, String indexName, String cid, String crawlDataFlag) { + try { + TransportClient client = getClient(clusterName); + System.out.println("**** " + indexName); + cid = cid.toLowerCase(); + DeleteByQueryRequestBuilder builder = DeleteByQueryAction.INSTANCE.newRequestBuilder(client) + .filter(QueryBuilders.termQuery(ESConstant.EN_SOURCE, cid)) + .filter(QueryBuilders.termQuery(ESConstant.CRAWLDATAFLAG, crawlDataFlag)) + .source(indexName); + BulkByScrollResponse response2 = builder.get(); + long deleted = response2.getDeleted(); + System.out.println(deleted); + } catch (Exception e) { + e.printStackTrace(); + } + } } diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils2.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils2.java index 56e5ed5..68e99b6 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils2.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils2.java @@ -4,7 +4,6 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.RateLimiter; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; @@ -33,7 +32,6 @@ import org.slf4j.LoggerFactory; import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; -import javax.annotation.PostConstruct; import java.net.InetAddress; import java.util.List; import java.util.Map; diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/MonitorUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/MonitorUtils.java index efe4d96..fa1df78 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/MonitorUtils.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/MonitorUtils.java @@ -8,10 +8,7 @@ import org.elasticsearch.search.SearchHits; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Set; public class MonitorUtils { diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java index ddcf5b2..87af8d9 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java @@ -7,7 +7,6 @@ import com.bfd.mf.common.web.entity.mysql.cache.Cluster; import com.bfd.mf.common.web.vo.params.QueryRequest; import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity; import com.bfd.nlp.common.util.object.TObjectUtils; -import com.bfd.nlp.common.util.string.TStringUtils; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -16,7 +15,10 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; @Component @@ -129,11 +131,11 @@ public class SliceScrollUtil { List currentIndexList = null; if(null != queryRequest.getSubjectId() && !("all").equals(queryRequest.getSubjectId())){ logger.info("查询 【专题数据】 subjectId :{}" ,queryRequest.getSubjectId()); - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111 + cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); }else{ logger.info("[SliceScrollUtil] fetchResultSubjectCache : 查询 【全局数据】"); - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109 + cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); currentIndexList = subjectQueryDataService.getIndexListByTimeRange(cluster, queryRequest.getStartTime(),queryRequest.getEndTime()); } diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/utility/DateUtil.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/utility/DateUtil.java index 80b21fb..667dd0e 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/utility/DateUtil.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/utility/DateUtil.java @@ -19,13 +19,11 @@ package com.bfd.mf.common.util.utility; import com.bfd.nlp.common.util.string.TStringUtils; import java.io.UnsupportedEncodingException; -import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; -import java.util.TimeZone; public class DateUtil { diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/cache/Cluster.java b/cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/cache/Cluster.java index 83b4190..7254667 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/cache/Cluster.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/cache/Cluster.java @@ -22,47 +22,37 @@ public class Cluster extends Model implements Serializable, Comparable @Override public int compareTo(Cluster o) { - if (this.getShardNum() >= o.getShardNum()) + if (this.getShardNum() >= o.getShardNum()) { return -1; - else + }else { return 1; + } } public static final class CLUSTER_TYPE { + + /** + * 日期索引（旧） + */ + public static final int normal_cluster_type = 0; // 172.18.1.134:9201 的集群 日期索引 SQ_Normal + /** + * 日期索引（新） + */ + public static final int newnormal_cluster_type = 1; // 172.18.1.134:9201 的集群 日期索引 SQ_Normal_2 /** - * 小集群 + * 专题索引（旧） */ - public static final int mini_cluster_type = 2; // 111 的ES 集群 专题索引 SQ_Mini + public static final int mini_cluster_type = 2; // 172.18.1.147:9213 的ES 集群 专题索引 SQ_Mini /** - * 大集群 + * 专题索引（新） */ - public static final int normal_cluster_type = 0; // 109 的集群 日期索引 SQ_Normal -// /** -// * 缓存 -// */ -// // public static final int subject_cluster_type = 2; // -// -// /** -// * 微博转发 -// */ -// public static final int weibo_cluster_type = 4; // 147的ES 集群 （前台打标后）SQ_Mini -// -// public static final int reply_cluster_type = 7; // 已废弃 （原160的ES） -// /** -// * 2期数据分析 集群 -// */ -// public static final int data_analysis_cluster_type = 5; // 158 的ES 集群（后台打标后的数据） SQ_Reputation_new - -// /** -// * 对应索引以 cl_company_* -// */ -// public static final int data_company_analysis_cluster_type = 6; // 已废弃 （原160的ES） + public static final int special_cluster_type = 4; // 172.18.1.81:9201 的ES 集群 专题索引 SQ_Mini_2 } public static Long getClusterIdByType(SubjectCluster subjectCluster, int type) { - if (type == Cluster.CLUSTER_TYPE.mini_cluster_type) { + if (type == CLUSTER_TYPE.mini_cluster_type) { return subjectCluster.getMiniClusterId(); - } else if (type == Cluster.CLUSTER_TYPE.normal_cluster_type) { + } else if (type == CLUSTER_TYPE.normal_cluster_type) { return subjectCluster.getNormalClusterId(); // } else if (type == Cluster.CLUSTER_TYPE.subject_cluster_type) { // return subjectCluster.getClusterId(); diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/topic/ParseExcelTask.java b/cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/topic/ParseExcelTask.java index 44086dd..8c87daa 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/topic/ParseExcelTask.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/topic/ParseExcelTask.java @@ -1,6 +1,5 @@ package com.bfd.mf.common.web.entity.mysql.topic; -import com.bfd.mf.common.web.entity.mysql.Model; import org.joda.time.DateTime; import javax.persistence.*; diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/UserRepository.java b/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/UserRepository.java deleted file mode 100644 index 20be0b9..0000000 --- a/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/UserRepository.java +++ /dev/null @@ -1,469 +0,0 @@ -///* -// * Copyright (C) 2016 Baifendian Corporation -// *

-// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// *

-// * http://www.apache.org/licenses/LICENSE-2.0 -// *

-// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package com.bfd.mf.common.web.repository.mysql; -// -//import com.bfd.mf.common.web.entity.mysql.User; -//import org.springframework.data.jpa.repository.Query; -//import org.springframework.data.repository.PagingAndSortingRepository; -//import org.springframework.data.repository.query.Param; -//import org.springframework.stereotype.Repository; -// -//@Repository -//public interface UserRepository extends PagingAndSortingRepository { -// -//// User findByUserNameAndDel(String userName, Boolean del); -//// -//// List findByAccountAndDel(String account, Boolean del); -//// -//// -//// @Query(value = "SELECT * FROM cl_user WHERE del = FALSE AND status = 1 AND account = ?1 ", nativeQuery = true) -//// User findAvailableByAccount(String account); -//// -//// @Query(value = "SELECT * FROM cl_user WHERE del = FALSE AND account = ?1 ", nativeQuery = true) -//// User findUserByAccount(String account); -//// -//// User findByUserNameAndPasswordAndDel(String userName, String pwd, Boolean del); -//// -//// User findByAccountAndPasswordAndDel(String account, String pwd, Boolean del); -//// -//// User findByCompanyIdAndUserNameAndDel(Long companyId, String userName, Boolean del); -//// -//// User findByCompanyIdAndAccountAndDel(Long companyId, String account, Boolean del); -// -// User findByTokenAndDel(String token, Boolean del); -// -//// @Query(value = "SELECT * FROM cl_user WHERE token = ?1 AND status = ?2", nativeQuery = true) -//// User findByTokenAndStatus(String token, int status); -//// -//// User findByTokenAndStatusAndDel(String token, int status, Boolean del); -// -// @Query(value = "SELECT company_id FROM cl_user WHERE token = :token AND del = :del", nativeQuery = true) -// Long getCompanyIdByTokenAndDel(@Param("token") String token, @Param("del") Boolean del); -// -// -//// @Query("select u from User u where u.roleId =?1 or u.roleId like '?1,%' or u.roleId like '%,?1' or u.roleId like " + -//// "'%,?1,%' and del = false") -//// List findByRoleId(String rid); -//// -//// List findByStatusAndCompanyIdAndDel(Integer status, Long companyId, Boolean del); -//// -//// List findByCompanyIdAndDel(Long companyId, Boolean del); -//// -//// -//// /** -//// * Get all users with role and creator information -//// * -//// * @param companyId -//// * @return -//// */ -//// @Query(value = "SELECT " + -//// "a.id , " + -//// "a.token , " + -//// "a.account , " + -//// "a.userName , " + -//// "r.id AS roleId , " + -//// "r.name as roleName, " + -//// "b.userName AS creator, " + -//// "a.status, " + -//// "a.email, " + -//// "a.wechat, " + -//// "a.wechatId, " + -//// "a.mobile, " + -//// "a.wechatVerified, " + -//// "a.wechatOpenId " + -//// "FROM " + -//// "User a , " + -//// "User b , " + -//// "Role r " + -//// "WHERE a.companyId = b.companyId " + -//// "AND a.companyId = :cid " + -//// "AND a.parentId = b.id " + -//// "AND a.roleId = r.id " + -//// "AND a.userName like :name " + -//// "AND (a.del IS NULL OR a.del = false)" + -//// "AND a.wechat IS NOT NULL " + -//// "AND a.wechatId is NOT NULL " + -//// "AND a.wechat <> '' " + -//// "AND a.wechatId <> '' " + -//// "AND a.wechatVerified in (:scope) " + -//// "order by a.createdTime desc") -//// Page findClientUserWeixinList(@Param("cid") Long companyId, @Param("name") String filterName, Pageable -//// pageable, @Param("scope") List scope); -//// -//// -//// /** -//// * Get all users with role and creator information -//// * -//// * @param companyId -//// * @return -//// */ -//// @Query(value = "SELECT " + -//// "a.id , " + -//// "a.token , " + -//// "a.account , " + -//// "a.userName , " + -//// "r.id AS roleId , " + -//// "r.name as roleName, " + -//// "b.userName AS creator, " + -//// "a.status, " + -//// "a.email, " + -//// "a.wechat, " + -//// "a.wechatId, " + -//// "a.mobile " + -//// "FROM " + -//// "User a , " + -//// "User b , " + -//// "Role r " + -//// "WHERE a.companyId = b.companyId " + -//// "AND a.companyId = :cid " + -//// "AND a.parentId = b.id " + -//// "AND a.roleId = r.id " + -//// "AND a.userName like :name " + -//// "AND (a.del IS NULL OR a.del = false)" + -//// "AND a.email IS NOT NULL " + -//// "AND a.email <> '' " + -//// "order by a.createdTime desc") -//// Page findClientUserEmailList(@Param("cid") Long companyId, @Param("name") String filterName, Pageable -//// pageable); -//// -//// @Modifying -//// @Query("update User set status = ?2, updatedTime = ?3 where id = ?1") -//// int switchUserStatus(Long userId, Integer status, Long updateTime); -//// -//// @Modifying -//// @Query("update User set del = ?2, updatedTime = ?3 where id = ?1") -//// int deleteUser(Long userId, Boolean isDel, Long updateTime); -//// -//// -////// @Query("SELECT " + -////// "a.id ," + -////// "a.account , " + -////// "a.userName, " + -////// "a.email, " + -////// "a.wechat, " + -////// "a.wechatId, " + -////// "a.mobile, " + -////// "r.id AS userGroupId , " + -////// "r.name as userGroupName, " + -////// "a.status, " + -////// "a.createdTime " + -////// "FROM " + -////// "User a , " + -////// "UserGroup r " + -////// "WHERE a.id = :uid " + -////// "AND a.userGroupId = r.id " + -////// "AND (a.del = :isDel ) " + -////// "order by a.createdTime desc") -////// List fetchUser(@Param("uid") Long userId, @Param("isDel") Boolean isDel); -//// -//// -//// /** -//// * Check if reach user create limitation, return Y if avaliable -//// *

-//// * Return NULL if not available -//// * -//// * @param token -//// * @return -//// */ -//// @Query(value = "SELECT \"Y\" AS VALID " + -//// "FROM " + -//// " (SELECT company_user_count AS u_limit " + -//// " FROM cl_company " + -//// " WHERE status = 1 " + -//// " AND id = " + -//// " (SELECT company_id " + -//// " FROM cl_user " + -//// " WHERE token = ?1)) a, " + -//// " (SELECT count(1) AS u_count " + -//// " FROM cl_user " + -//// " WHERE company_id = " + -//// " (SELECT company_id " + -//// " FROM cl_user " + -//// " WHERE token = ?1) " + -//// " AND del = FALSE) b " + -//// "WHERE a.u_limit >= b.u_count", nativeQuery = true) -//// List checkIfUserLimitReached(String token); -//// -//// /** -//// * Check if user over dead date and open status, return true if available -//// *

-//// * Return false if not available -//// * -//// * @param token -//// * @return -//// */ -//// @Query(value = -//// " SELECT \"Y\" AS flag " + -//// " FROM cl_company " + -//// " WHERE status = 1 " + -//// " AND deadline_time < UNIX_TIMESTAMP() " + -//// " AND id = " + -//// " (SELECT company_id " + -//// " FROM cl_user " + -//// " WHERE token = ?1 and del = false and status = 1)", nativeQuery = true) -//// List checkIfUserOverDeadDate(String token); -//// -//// /** -//// * 返回话题上限数 -//// *

-//// * SELECT company_topic_count FROM cl_company -//// * WHERE id=(SELECT company_id FROM cl_user WHERE token = '8FB24E550815BCBA') -//// * -//// * @param token -//// * @return -//// */ -//// @Query(value = -//// " SELECT company_topic_count " + -//// " FROM cl_company " + -//// " WHERE id = " + -//// " (SELECT company_id" + -//// " FROM cl_user " + -//// " WHERE token = ?1)", nativeQuery = true) -//// int getCompanyTopicCount(String token); -//// -//// /** -//// * 返回创建话题的ids -//// * -//// * @param token -//// * @return -//// */ -//// @Query(value = -//// " SELECT topic_ids" + -//// " FROM cl_role " + -//// " WHERE company_id = " + -//// " (SELECT company_id" + -//// " FROM cl_user " + -//// " WHERE token = ?1) ", nativeQuery = true) -//// List getSujectIds(String token); -//// -//// /** -//// * 返回客户创建的话题总数的结果 -//// *

-//// * SELECT COUNT(DISTINCT(label_id)) FROM mf_subject -//// * WHERE label_id IN( -//// * SELECT topic_ids FROM mf_role -//// * WHERE company_id ='14') -//// * -//// * @param labelIds -//// * @return -//// */ -//// @Query(value = -//// " SELECT count(1) " + -//// " FROM cl_subject " + -//// " WHERE label_id IN " + -//// " (?1)", nativeQuery = true) -//// int getCompanyResult(Set labelIds); -//// -//// @Query("select u from User u, Role r " + -//// "where u.roleId = r.id and u.companyId =:cid and " + -//// "r.isSystem >= (select r2.isSystem from Role r2 where r2.id = :rid) and u.del = false") -//// List getUserListByCompanyAndRole(@Param("cid") Long companyId, @Param("rid") Long roleId); -//// -//// @Query("select " + -//// "u.account," + -//// "wu.openId," + -//// "wu.subscribe," + -//// "wu.nickName," + -//// "wu.sex," + -//// "wu.language," + -//// "wu.city," + -//// "wu.province," + -//// "wu.headimgUrl," + -//// "wu.subscribeTime," + -//// "wu.unionId," + -//// "wu.remark," + -//// "wu.grouId," + -//// "wu.tagIdList," + -//// "wu.mfRemark " + -//// "from User u, Role r, WechatBind wb, WechatUser wu " + -//// "where u.roleId = r.id and u.companyId =:cid and u.id = wb.userId and wb.openId = wu.openId and " + -//// "r.isSystem >= (select r2.isSystem from Role r2 where r2.id = :rid) and u.del = false") -//// List getUserInfoListByCompanyAndRole(@Param("cid") Long companyId, @Param("rid") Long roleId); -//// -//// @Query(value = "select * from cl_user where id =?1", nativeQuery = true) -//// User getUserById(long creator); -//// -//// /** -//// * 返回客户ID -//// * -//// * @param token -//// * @return -//// */ -//// @Query(value = "select company_id from cl_user where token =?1", nativeQuery = true) -//// Integer findCompanyIdByToken(String token); -//// -//// /** -//// * 返回有效客户 -//// * -//// * @param id -//// * @param flag -//// * @return -//// */ -//// @Query(value = "SELECT token FROM cl_user WHERE company_id=?1 AND del=?2", nativeQuery = true) -//// List findTokenByCompanyId(Long id, boolean flag); -//// -//// @Query(value = "SELECT token FROM cl_user WHERE company_id =?1 AND del=?2 AND STATUS=?3 limit 1", nativeQuery = true) -//// String findEffectTokenByCompanyIdAndDel(Long companyId, boolean del, int status); -//// -//// @Query(value = -//// "select company_id " + -//// "from cl_user " + -//// "where token = ?1 and del=?2", nativeQuery = true) -//// Long findCompanyIdByTokenAndDel(String token, boolean del); -//// -//// @Query(value = "SELECT account FROM cl_user WHERE email=?1 AND status=?2", nativeQuery = true) -//// String fetchUserOfAccountByReceiverAndStatus(String receiver, int status); -//// -//// @Query(value = -//// " SELECT company_keyword_count " + -//// " FROM cl_company " + -//// " WHERE id = " + -//// " (SELECT company_id" + -//// " FROM cl_user " + -//// " WHERE token = ?1)", nativeQuery = true) -//// int getCompanyKeywordCount(String token); -//// -//// @Query(value = "SELECT company_exclude_count FROM cl_company WHERE id = (SELECT company_id FROM cl_user WHERE token = ?1)", nativeQuery = true) -//// int getCompanyExcludeCount(String token); -//// -//// @Query(value = "SELECT role_id FROM cl_user WHERE token = :token AND del = :del", nativeQuery = true) -//// String getRoleIdByTokenAndDel(@Param("token") String token, @Param("del") Boolean del); -//// -//// /** -//// * Get all users with role and creator information -//// * -//// * @param companyId -//// * @return -//// */ -//// @Query(value = "SELECT " + -//// "a.id , " + -//// "a.account , " + -//// "a.userName , " + -//// "r.id AS userGroupId , " + -//// "r.name as userGroupName, " + -//// //"b.userName AS creator, " + -//// " (case when a.parentId = 0 then '系统' when m.roleType = 1 then '管理员' else b.account end ) AS creator , " + -//// "a.status, " + -//// "a.email, " + -//// "a.wechat, " + -//// "a.wechatId, " + -//// "a.mobile, " + -//// "a.createdTime, "+ -//// "r.roleType "+ -//// "FROM " + -//// "User a , " + -//// "User b , " + -//// "UserGroup m , " + -//// "UserGroup r " + -//// "WHERE " + -//// //"a.companyId = b.companyId " + -//// " a.companyId = :cid " + -//// "AND a.userGroupId = r.id " + -//// "AND a.userGroupId = :ugid " + -//// "AND b.userGroupId = m.id " + -//// "AND a.parentId = b.id " + -//// "AND a.isShadow <> :isShadow " + -//// //"AND a.roleId = r.id " + -//// "AND (a.userName like :name or a.account like :name) " + -//// "AND (a.del = :isDel) " + -//// "order by a.createdTime desc" ) -//// Page findClientUserListByGroupId(@Param("cid") Long companyId, @Param("name") String filterName, Pageable pageable, @Param("ugid") Long userGroupId, @Param("isDel") Boolean isDel, @Param("isShadow") Integer isShadow); -//// -//// @Query(value = "SELECT * FROM cl_user WHERE account = ?1", nativeQuery = true) -//// User findByAccount(String account); -//// -//// @Query(value = "SELECT * FROM cl_user WHERE user_group_id = ?1 and del = ?2", nativeQuery = true) -//// List findByUserGroupId(String userGroupId, Boolean del); -//// -//// @Query(value = "SELECT token FROM cl_user WHERE id IN (?1)", nativeQuery = true) -//// List getTokenListByIdList(List idList); -//// -//// @Query(value = "SELECT id FROM cl_user WHERE token IN (?1)", nativeQuery = true) -//// List getIdListByTokenList(List tokenList); -//// -//// @Query(value = "SELECT " + -//// "a.id , " + -//// "a.account , " + -//// "a.userName , " + -//// "r.id AS userGroupId , " + -//// "r.name as userGroupName, " + -//// // "b.userName AS creator, " + -//// " (case when a.parentId = 0 then '系统' when m.roleType = 1 then '管理员' else b.account end ) AS creator , " + -//// "a.status, " + -//// "a.email, " + -//// "a.wechat, " + -//// "a.wechatId, " + -//// "a.mobile, " + -//// "a.createdTime, " + -//// "r.roleType " + -//// "FROM " + -//// "User a , " + -//// "User b , " + -//// "UserGroup m , " + -//// "UserGroup r " + -//// "WHERE " + -//// //"a.companyId = b.companyId " + -//// "a.companyId = :cid " + -//// "AND a.parentId = b.id " + -//// "AND a.userGroupId = r.id " + -//// "AND b.userGroupId = m.id " + -//// "AND a.isShadow <> :isShadow " + -//// "AND r.roleType not in :ugrt " + -//// "AND (a.userName like :name or a.account like :name) " + -//// "AND (a.del = :isDel ) " + -//// "order by a.createdTime desc") -//// Page findClientUserListFilterByRole(@Param("cid") Long companyId, -//// @Param("name") String filterName, -//// Pageable pageable, -//// @Param("isDel") Boolean isDel, -//// @Param("ugrt") List userGroupRoleType, -//// @Param("isShadow") Integer isShadow); -// -//// @Query(value = "SELECT id,account,user_group_id FROM cl_user WHERE company_id = ?1 and del = ?2 ORDER BY created_time DESC ", nativeQuery = true) -//// List findAllByCompanyId(Long companyId, boolean del); -//// -//// @Query(value = "SELECT * FROM cl_user WHERE id = ?1 AND status = ?2 AND del = ?3", nativeQuery = true) -//// User findByIdAndStatusAndDel(Long userId, Integer userNormal, Boolean userNotDelete); -//// -//// @Query(value = "SELECT * FROM cl_user WHERE token = ?1 ", nativeQuery = true) -//// User findBytoken(String creator); -// -//// @Query("SELECT " + -//// "a.id ," + -//// "a.account , " + -//// "a.userName, " + -//// "a.email, " + -//// "a.wechat, " + -//// "a.wechatId, " + -//// "a.mobile, " + -//// "r.id AS userGroupId , " + -//// "r.name as userGroupName, " + -//// "a.status, " + -//// "a.createdTime, " + -//// "a.password " + -//// "FROM " + -//// "User a , " + -//// "UserGroup r " + -//// "WHERE a.token = :token " + -//// "AND a.userGroupId = r.id " + -//// "AND (a.del = :isDel ) " + -//// "order by a.createdTime desc") -//// List fetchUserByTokenAndDel(@Param("token") String token, @Param("isDel") Boolean userNotDelete); -// -//// @Query(value = "select g.channel_ids from cl_user u, cl_user_group g where u.token = ?1 AND g.id = u.user_group_id ", nativeQuery = true) -//// String getChannelIdsByToken(String token); -//} diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/topic/ParseExcelTaskRepository.java b/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/topic/ParseExcelTaskRepository.java index 7144943..e6ae3b4 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/topic/ParseExcelTaskRepository.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/topic/ParseExcelTaskRepository.java @@ -1,7 +1,6 @@ package com.bfd.mf.common.web.repository.mysql.topic; import com.bfd.mf.common.web.entity.mysql.topic.ParseExcelTask; -import com.bfd.mf.common.web.entity.mysql.topic.Subject; import org.apache.ibatis.annotations.Options; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/QueryRequest.java b/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/QueryRequest.java index 7310cc9..ac25ae2 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/QueryRequest.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/QueryRequest.java @@ -19,6 +19,7 @@ package com.bfd.mf.common.web.vo.params; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; + import java.io.Serializable; import java.util.ArrayList; import java.util.List; @@ -27,7 +28,6 @@ import java.util.List; public class QueryRequest implements Serializable { private static final long serialVersionUID = 1L; - // 必传字段 // private Long subjectId; @ApiModelProperty(value = "subjectId" , required = true , notes = "专题ID") diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataLineCount.java b/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataLineCount.java deleted file mode 100644 index 0cef43a..0000000 --- a/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataLineCount.java +++ /dev/null @@ -1,88 +0,0 @@ -///* -// * Copyright (C) 2016 Baifendian Corporation -// *

-// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// *

-// * http://www.apache.org/licenses/LICENSE-2.0 -// *

-// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package com.bfd.mf.common.web.vo.view.analysis; -// -///** -// * 折线图统计 -// * -// * @author quanyou.chang -// * @create 2016-09-22 10:43 -// **/ -//public class DataLineCount extends DataCountBase implements Comparable { -// /** -// * 时间戳 -// */ -// private Long timestamp; -// /** -// * 值 -// */ -// private Long data; -// -// public Long getTimestamp() { -// return timestamp; -// } -// -// public void setTimestamp(Long timestamp) { -// this.timestamp = timestamp; -// } -// -// -// public String getName() { -// return name; -// } -// -// public void setName(String name) { -// this.name = name; -// } -// -// /** -// * 坐标 -// */ -// private String name; -// -// -// public Long getData() { -// return data; -// } -// -// public void setData(Long data) { -// this.data = data; -// } -// -// @Override -// public int compareTo(DataLineCount o) { -// return this.getTimestamp().compareTo(o.getTimestamp()); -// } -// @Override -// public boolean equals(Object o) { -// if (this == o) return true; -// if (o == null || getClass() != o.getClass()) return false; -// -// DataLineCount that = (DataLineCount) o; -// -// if (timestamp != null ? !timestamp.equals(that.timestamp) : that.timestamp != null) return false; -// return name != null ? name.equals(that.name) : that.name == null; -// -// } -// -// @Override -// public int hashCode() { -// int result = timestamp != null ? timestamp.hashCode() : 0; -// result = 31 * result + (name != null ? name.hashCode() : 0); -// return result; -// } -//} diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorBaseEntity.java b/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorBaseEntity.java index 169e784..468fd44 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorBaseEntity.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorBaseEntity.java @@ -17,14 +17,12 @@ package com.bfd.mf.common.web.vo.view.monitor; import com.alibaba.fastjson.JSONObject; -import com.bfd.mf.common.util.constants.ESConstant; import com.google.common.base.Objects; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import java.io.Serializable; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,6 +35,8 @@ public class ESMonitorBaseEntity implements Comparable, Ser private String author; // 作者 private String authorId; // 作者ID private String avatar; // 作者头像 + private String authornickname; // 作者昵称 + private String userType; private String title; // 文章title private Long pubTime = 0L; //文章发布时间 @@ -65,14 +65,29 @@ public class ESMonitorBaseEntity implements Comparable, Ser private String commentId; private String content; // 内容 - private String contentSimHash; //文章SimHash - private Integer contentSize; //正文长度 - // 转、赞、评 数字 + private String forwardContent; + /** + * 文章SimHash + */ + private String contentSimHash; + /** + * //正文长度 + */ + private Integer contentSize; + /** + * 转、赞、评 ,收藏数、阅读数、好看数的 数字 + */ + private String quoteCount ; private String attitudesCount; private Integer commentsCount = 0; private String collectCount; - // 词云 + private int readCount; + private int viewCnt; + + /** + * 词云 + */ private List hlKeyWords; private List places; private List hashTag; @@ -122,42 +137,101 @@ public class ESMonitorBaseEntity implements Comparable, Ser private String promotionInfo; - private int readCount; - public int getReadCount() { - return readCount; + private String forumScore; + private String userTypeContent; + private String pageType; + + private int hasImage; + private int hasVideo; + private int hasFile; + + public int getViewCnt() { + return viewCnt; } - public void setReadCount(int readCount) { - this.readCount = readCount; + public void setViewCnt(int viewCnt) { + this.viewCnt = viewCnt; + } + + public int getHasImage() { + return hasImage; + } + + public void setHasImage(int hasImage) { + this.hasImage = hasImage; + } + + public int getHasVideo() { + return hasVideo; + } + + public void setHasVideo(int hasVideo) { + this.hasVideo = hasVideo; + } + + public int getHasFile() { + return hasFile; + } + + public void setHasFile(int hasFile) { + this.hasFile = hasFile; } - private int ugc; - private int egc; - private int pgc; + public String getPageType() { + return pageType; + } + + public void setPageType(String pageType) { + this.pageType = pageType; + } + + public String getUserTypeContent() { + return userTypeContent; + } + + public void setUserTypeContent(String userTypeContent) { + this.userTypeContent = userTypeContent; + } + + public String getUserType() { + return userType; + } + + public void setUserType(String userType) { + this.userType = userType; + } - public int getUgc() { - return ugc; + public String getForumScore() { + return forumScore; } - public void setUgc(int ugc) { - this.ugc = ugc; + public void setForumScore(String forumScore) { + this.forumScore = forumScore; } - public int getEgc() { - return egc; + public String getForwardContent() { + return forwardContent; } - public void setEgc(int egc) { - this.egc = egc; + public void setForwardContent(String forwardContent) { + this.forwardContent = forwardContent; } - public int getPgc() { - return pgc; + public int getReadCount() { + return readCount; + } + + public void setReadCount(int readCount) { + this.readCount = readCount; } - public void setPgc(int pgc) { - this.pgc = pgc; + public String getAuthornickname() { + return authornickname; + } + + public void setAuthornickname(String authornickname) { + this.authornickname = authornickname; } public String getPromotionInfo() { @@ -687,15 +761,16 @@ public class ESMonitorBaseEntity implements Comparable, Ser } public String getSysSentimentTag() { - if(sysSentiment < 0.5){ - sysSentimentTag = "负面"; - } - if(sysSentiment == 0.5){ - sysSentimentTag = "中性"; - } - if(sysSentiment > 0.5){ - sysSentimentTag = "正面"; - } + sysSentimentTag = "中性"; +// if(sysSentiment < 0.5){ +// sysSentimentTag = "负面"; +// } +// if(sysSentiment == 0.5){ +// sysSentimentTag = "中性"; +// } +// if(sysSentiment > 0.5){ +// sysSentimentTag = "正面"; +// } return sysSentimentTag; } diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorEntity.java b/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorEntity.java index 86e167b..65c8799 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorEntity.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorEntity.java @@ -20,8 +20,6 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; public class ESMonitorEntity extends ESMonitorBaseEntity implements Serializable { diff --git a/cl_search_api/src/main/java/com/bfd/mf/controller/SearchAnalysisController.java b/cl_search_api/src/main/java/com/bfd/mf/controller/SearchAnalysisController.java index c44abcc..2557438 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/controller/SearchAnalysisController.java +++ b/cl_search_api/src/main/java/com/bfd/mf/controller/SearchAnalysisController.java @@ -1,10 +1,10 @@ package com.bfd.mf.controller; import com.alibaba.fastjson.JSONObject; -import com.bfd.mf.common.web.vo.params.QueryRequest; -import com.bfd.mf.service.SearchAnalysisService; import com.bfd.mf.common.util.enums.RTCodeEnum; import com.bfd.mf.common.web.component.wrapper.ResponseWrapper; +import com.bfd.mf.common.web.vo.params.QueryRequest; +import com.bfd.mf.service.SearchAnalysisService; import com.bfd.nlp.common.util.constants.MediaTypes; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -12,7 +12,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; @Controller diff --git a/cl_search_api/src/main/java/com/bfd/mf/controller/SearchAuthorController.java b/cl_search_api/src/main/java/com/bfd/mf/controller/SearchAuthorController.java index 6404291..4cdbdb2 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/controller/SearchAuthorController.java +++ b/cl_search_api/src/main/java/com/bfd/mf/controller/SearchAuthorController.java @@ -38,7 +38,8 @@ public class SearchAuthorController { try { JSONObject result; String scorllId = queryRequest.getScrollId(); - if(null != scorllId){ // 这个是导出要用的 + // 这个是导出要用的 + if(null != scorllId){ result = searchDataService.exportDataInSubjectIndex(queryRequest); return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result); }else{ // 这个是查询 diff --git a/cl_search_api/src/main/java/com/bfd/mf/controller/SearchDataController.java b/cl_search_api/src/main/java/com/bfd/mf/controller/SearchDataController.java index e24de10..90210d1 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/controller/SearchDataController.java +++ b/cl_search_api/src/main/java/com/bfd/mf/controller/SearchDataController.java @@ -2,24 +2,31 @@ package com.bfd.mf.controller; import com.alibaba.fastjson.JSONObject; import com.bfd.mf.common.util.constants.ESConstant; -import com.bfd.mf.common.web.vo.params.QueryRequest; -import com.bfd.mf.service.SearchDataService; import com.bfd.mf.common.util.enums.RTCodeEnum; import com.bfd.mf.common.web.component.wrapper.ResponseWrapper; +import com.bfd.mf.common.web.vo.params.QueryRequest; +import com.bfd.mf.service.SearchDataService; import com.bfd.mf.service.UpdateService; +import com.bfd.nlp.common.util.constants.MediaTypes; import io.swagger.annotations.Api; import io.swagger.annotations.ApiImplicitParam; import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.*; -import io.swagger.annotations.ApiOperation; -import com.bfd.nlp.common.util.constants.MediaTypes; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; import java.util.ArrayList; +/** + * @author dujing + */ + @Controller @RequestMapping("/crawl") @Api(value="数据查询的控制器") @@ -56,9 +63,11 @@ public class SearchDataController { } return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result); }else { - if (subjectId.equals("all") || subjectId.contains(",")) {// 全局数据查询 + // 全局数据查询 + if (subjectId.equals("all") || subjectId.contains(",")) { result = searchDataService.queryDataList(queryRequest); - } else if (subjectId.equals("")) { // 没有专题 + // 没有专题 + } else if (subjectId.equals("")) { result.put(ESConstant.ALLDOCNUMBER, 0L); result.put(ESConstant.MONITORLISTS, new ArrayList<>()); } else { @@ -68,11 +77,11 @@ public class SearchDataController { } Integer allDocNumber = result.getIntValue(ESConstant.ALLDOCNUMBER); Integer limit = queryRequest.getLimit(); - Integer page = 1; - if(allDocNumber%limit==0){ + int page = 1; + if(allDocNumber%limit == 0){ page = allDocNumber/limit; }else{ - page = allDocNumber/limit +1; + page = allDocNumber/limit + 1; } if(null != queryRequest.getPage() && !queryRequest.getPage().equals("")) { if (page > 0 && queryRequest.getPage() > page) { @@ -83,7 +92,7 @@ public class SearchDataController { logger.info("接口查询时长：statr:"+ start +" ; end:"+end + " ; time = " + (end - start) + " ; count = "+result.get(ESConstant.ALLDOCNUMBER)); return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result); } catch (Exception e) { - logger.error("[queryData] Failed,The error message is :{}", e); + logger.error("[SearchDataController] queryDataList Failed,The error message is :{}", e); return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed"); } @@ -98,8 +107,7 @@ public class SearchDataController { @ApiImplicitParams({ @ApiImplicitParam(paramType="query", name = "subjectId", value = "专题ID", required = true, dataType = "String"), @ApiImplicitParam(paramType="query", name = "docId", value = "主贴唯一ID", required = true, dataType = "String"), - @ApiImplicitParam(paramType="query", name = "siteId", value = "站点ID", required = true, dataType = "String"), - }) + @ApiImplicitParam(paramType="query", name = "siteId", value = "站点ID", required = true, dataType = "String"),}) public JSONObject getInfo(String subjectId,String docId,String siteId) { QueryRequest queryRequest = new QueryRequest(); queryRequest.setSubjectId(subjectId); @@ -154,14 +162,23 @@ public class SearchDataController { } - + /** + * 查询 数据的Counts 用户左侧的显示 + * @param queryRequest + * @return + */ @ApiOperation(value = "查询数据列表") @RequestMapping(value = "/subject/queryCounts", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8) @ResponseBody public JSONObject queryDataCounts(@RequestBody QueryRequest queryRequest) { logger.info("[queryDataCounts] partial / Params: {}", JSONObject.toJSONString(queryRequest)); try { - JSONObject result = searchDataService.queryDataCountsInOneIndex(queryRequest); + JSONObject result = new JSONObject(); + if(null != queryRequest.getSubjectId()) { + result = searchDataService.queryDataCountsInOneIndex(queryRequest); + }else{ + return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed"); + } return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result); } catch (Exception e) { logger.error("[queryDataCounts] Failed,The error message is :{}", e); @@ -192,20 +209,26 @@ public class SearchDataController { } + /** + * 示例文件夹的专题复制 + */ @ApiOperation(value = "复制专题") @RequestMapping(value = "/reindex/reindexSubject", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8) @ResponseBody public JSONObject reIndex(@RequestBody QueryRequest queryRequest){ logger.info("[reIndex] partial / Params: {}", JSONObject.toJSONString(queryRequest)); try { - JSONObject result = searchDataService.reIndexSubject(queryRequest); - return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result); + searchDataService.reIndexSubject(queryRequest); } catch (Exception e) { logger.error("[reIndex] Failed,The error message is :{}", e); return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed"); } + return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, new JSONObject()); } + /** + * 删除专题的接口 + */ @ApiOperation(value = "删除专题") @RequestMapping(value = "/delete/deleteBySubjectId", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8) @ResponseBody @@ -220,4 +243,74 @@ public class SearchDataController { } } + @ApiOperation(value = "根据 cid 删除指定专题下的数据") + @RequestMapping(value = "/delete/deleteByCid", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8) + @ResponseBody + public JSONObject deleteSubjectByCid(@RequestBody QueryRequest queryRequest){ + logger.info("[deleteSubject] partial / Params: {}", JSONObject.toJSONString(queryRequest)); + try { + JSONObject result = searchDataService.deleteBySubjectIdByCid(queryRequest); + return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result); + } catch (Exception e) { + logger.error("[deleteSubject] Failed,The error message is :{}", e); + return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed"); + } + } + + @ApiOperation(value = "根据 crawlDataFlag 删除指定专题下的问题") + @RequestMapping(value = "/delete/deleteByCrawlDataFlag", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8) + @ResponseBody + public JSONObject deleteSubjectByCrawlDataFlag(@RequestBody QueryRequest queryRequest){ + logger.info("[deleteSubject] partial / Params: {}", JSONObject.toJSONString(queryRequest)); + try { + JSONObject result = searchDataService.deleteBySubjectIdByCrawlDataFlag(queryRequest); + return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result); + } catch (Exception e) { + logger.error("[deleteSubject] Failed,The error message is :{}", e); + return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed"); + } + } + + + /** + * 获取json 结构数据 + * @param queryRequest + * @return + */ + @ApiOperation(value = "查询数据列表") + @RequestMapping(value = "/subject/getJson", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8) + @ResponseBody + public JSONObject queryDataJsonList(@RequestBody QueryRequest queryRequest) { + logger.info("[queryDataList] partial / Params: {}", JSONObject.toJSONString(queryRequest)); + try { + JSONObject result = new JSONObject(); + long start = System.currentTimeMillis(); + String scorllId = queryRequest.getScrollId(); + + // 专题数据导出 + result = searchDataService.exportJsonDataInSubject(queryRequest); + + Integer allDocNumber = result.getIntValue(ESConstant.ALLDOCNUMBER); + Integer limit = queryRequest.getLimit(); + Integer page = 1; + if(allDocNumber%limit==0){ + page = allDocNumber/limit; + }else{ + page = allDocNumber/limit +1; + } + if(null != queryRequest.getPage() && !queryRequest.getPage().equals("")) { + if (page > 0 && queryRequest.getPage() > page) { + return ResponseWrapper.buildResponse(RTCodeEnum.C_SUBJECT_GRAMMAR_ERROR, "总数和分页不匹配"); + } + } + long end = System.currentTimeMillis(); + logger.info("接口查询时长：statr:"+ start +" ; end:"+end + " ; time = " + (end - start) + " ; count = "+result.get(ESConstant.ALLDOCNUMBER)); + return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result); + } catch (Exception e) { + logger.error("[queryData] Failed,The error message is :{}", e); + return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed"); + } + + } + } diff --git a/cl_search_api/src/main/java/com/bfd/mf/controller/UploadExcelController.java b/cl_search_api/src/main/java/com/bfd/mf/controller/UploadExcelController.java index aa5858d..3b3215a 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/controller/UploadExcelController.java +++ b/cl_search_api/src/main/java/com/bfd/mf/controller/UploadExcelController.java @@ -12,7 +12,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.multipart.MultipartFile; import java.io.File; @@ -37,8 +40,7 @@ public class UploadExcelController { @ApiOperation(value = "上传Excel", httpMethod = "POST") @PostMapping(value = "/SubjectsExcel") @ResponseBody - public JSONObject insertExcelTask(@RequestParam("file") MultipartFile file, - @RequestParam("userId") String userId) { + public JSONObject insertExcelTask(@RequestParam("file") MultipartFile file, @RequestParam("userId") String userId) { logger.info("[insertExcelTask] partial / Params: {}", userId); try { JSONObject jsonObject = new JSONObject(); diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchAnalysisService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchAnalysisService.java index 005b44c..27d24ce 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchAnalysisService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchAnalysisService.java @@ -95,9 +95,12 @@ public class SearchAnalysisService { Map valueLabelMap = new HashMap<>(); Map categoryLabelMap = new HashMap<>(); for (ESMonitorEntity esMonitorEntity : esMonitorEntityList) { - String docType = esMonitorEntity.getDocType(); // 页面类型 - String valueLabel = esMonitorEntity.getValueLabel(); // 价值标签 - String categoryLabel = esMonitorEntity.getCategoryLabel();// 分类标签 + // 页面类型 + String docType = esMonitorEntity.getDocType(); + // 价值标签 + String valueLabel = esMonitorEntity.getValueLabel(); + // 分类标签 + String categoryLabel = esMonitorEntity.getCategoryLabel(); channelMap.put(docType,getDocTypeNum(docType)); valueLabelMap.put(valueLabel,valueLabel); categoryLabelMap.put(categoryLabel,categoryLabel); diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java index a16d02f..107d181 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java @@ -19,7 +19,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; @Service public class SearchAuthorService extends CrudService implements Serializable { diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java index 0f1b943..3e726b6 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java @@ -24,8 +24,8 @@ import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; -import springfox.documentation.spring.web.json.Json; import java.io.Serializable; import java.util.*; @@ -129,7 +129,8 @@ public class SearchDataService extends CrudService siteOtherMap = siteMap.get(enSource); if (siteOtherMap.containsKey("site_id")) { siteId = siteMap.get(enSource).get("site_id").toString(); @@ -243,7 +258,11 @@ public class SearchDataService extends CrudService hlKeywords = new ArrayList<>(); @@ -435,28 +459,51 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>(); String subjectId = queryRequest.getSubjectId(); if(null != queryRequest.getSubjectId() && !("").equals(subjectId)){ // 如果是专题，去专题的索引查就行 - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111 + cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId; currentIndexList.add(subjectId); }else{ // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来 @@ -628,10 +685,10 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>(); String subjectId = queryRequest.getSubjectId(); if(!("").equals(subjectId) && null != subjectId){ // 如果是专题，去专题的索引查就行 - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111 + cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId; }else{ // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来 logger.info("[SearchDataService] queryOneDataByDocId 查询 全局数据 : {}" , subjectId); @@ -670,7 +727,7 @@ public class SearchDataService extends CrudService>imagePathSize = (List>) jsonObject.get(ESConstant.IMAGEPATHSIZE); for (Map imagePath: imagePathSize) { String url = imagePath.get(ESConstant.URL); @@ -678,7 +735,7 @@ public class SearchDataService extends CrudService>videoPathSize = (List>) jsonObject.get(ESConstant.VIDEOPATHSIZE); for (Map videoPath: videoPathSize) { String url = videoPath.get(ESConstant.URL); @@ -705,9 +762,13 @@ public class SearchDataService extends CrudService smallImgs = (List) jsonObject.get(ESConstant.SMALLIMGS); - List> imagePathSize = new ArrayList<>(); - if(null != smallImgs && smallImgs.size() > 0) { // 由于电商的图片不做下载，因此输出到页面的时候写死了大小和分辨率 - for (Object img : smallImgs) { - if (!img.toString().contains("http")) { - Map imagePathMap = new HashMap<>(); - String url = "http:" + img ; - imagePathMap.put(ESConstant.URL ,url); - imagePathMap.put(ESConstant.SIZE ,"4KB"); - imagePathMap.put(ESConstant.VIDEOTIME ,""); - imagePathMap.put(ESConstant.RESOLUTION ,"50×50"); - imagePathSize.add(imagePathMap); - } - } - } - jsonObject.put(ESConstant.IMAGEPATHSIZE,imagePathSize); - } } return jsonObject; } @@ -798,12 +841,15 @@ public class SearchDataService extends CrudService0) { Map result = searchDataResponse.getHits().getHits()[0].getSourceAsMap(); - if (result.size() > 0) { - result.entrySet().stream() - .forEach(entry -> { - if (entry.getKey().equals(ESConstant.FILEPATH) && entry.getValue().equals("")) { - jsonObject.put(entry.getKey(), new ArrayList<>()); - } else if (entry.getKey().equals(ESConstant.FILEPATHSIZE) || - entry.getKey().equals(ESConstant.IMAGEPATHSIZE) || - entry.getKey().equals(ESConstant.VIDEOPATHSIZE)) { - if(entry.getValue().toString() .contains(ESConstant.URL)) { - jsonObject.put(entry.getKey(), JSONObject.parseArray(entry.getValue().toString())); - }else{ - jsonObject.put(entry.getKey(),new ArrayList<>()); - } - } else if(entry.getKey().equals(ESConstant.ATTITUDES_COUNT)){ - if(entry.getValue().toString().contains(ESConstant.TOTALCOUNT)){ - JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString()); - jsonObject.put(entry.getKey(),totalCount.get(ESConstant.TOTALCOUNT)); - } - } else { + if (result.size() > 0) { result.entrySet().stream() + .forEach(entry -> { + if (entry.getKey().equals(ESConstant.FILEPATH) && entry.getValue().equals("")) { + jsonObject.put(entry.getKey(), new ArrayList<>()); + } else if (entry.getKey().equals(ESConstant.FILEPATHSIZE) + || entry.getKey().equals(ESConstant.IMAGEPATHSIZE) + || entry.getKey().equals(ESConstant.VIDEOPATHSIZE)) { + if(entry.getValue().toString() .contains(ESConstant.URL)) { jsonObject.put(entry.getKey(), entry.getValue()); + }else{ + jsonObject.put(entry.getKey(),new ArrayList<>()); } - }); + } else if(entry.getKey().equals(ESConstant.ATTITUDES_COUNT)){ + if(entry.getValue().toString().contains(ESConstant.TOTALCOUNT)){ + JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString()); + jsonObject.put(entry.getKey(),totalCount.get(ESConstant.TOTALCOUNT)); + }else{ + jsonObject.put(entry.getKey(),entry.getValue()); + } + } else { + jsonObject.put(entry.getKey(), entry.getValue()); + } + }); } } return jsonObject; @@ -865,7 +912,7 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>(); // 获取ES的参数及要查询的索引列表 String subjectId = queryRequest.getSubjectId(); - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111 + cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 special_cluster_type currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster,subjectId); Long clusterId = cluster.getId(); logger.info("[SearchDataService] queryDataList clusterId : {}; currentIndexList : {}" ,clusterId, currentIndexList.toString()); @@ -947,7 +994,7 @@ public class SearchDataService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); Long clusterId = cluster.getId(); logger.info("[SearchDataService] queryDataInOneIndex: clusterId :{} ; currentIndexList : {}", clusterId,currentIndexList.toString()); @@ -977,8 +1024,8 @@ public class SearchDataService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); Long clusterId = cluster.getId(); String [] indexName = currentIndexList.toArray(new String[currentIndexList.size()]); @@ -996,6 +1043,22 @@ public class SearchDataService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); + Long clusterId = cluster.getId(); + String [] indexName = currentIndexList.toArray(new String[currentIndexList.size()]); + logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; clusterId :{} ; currentIndexList :{}",indexName[0], clusterId , currentIndexList.toString()); + jsonObject= esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest); + }catch (Exception e){ + e.printStackTrace(); + } + return jsonObject; + } + public JSONObject exportDataInDateIndex(QueryRequest queryRequest) { JSONObject jsonObject = new JSONObject(); try { @@ -1007,7 +1070,7 @@ public class SearchDataService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); Long clusterId = cluster.getId(); logger.info("[SearchDataService] queryDataCountsInOneIndex: clusterId : {}; currentIndexList : {}" , clusterId ,currentIndexList.toString()); @@ -1056,10 +1119,17 @@ public class SearchDataService extends CrudService keyWordsMaps = new HashMap<>(); - Map placesWordsMaps = new HashMap<>(); // 地点 - Map emojiWordsMaps = new HashMap<>(); // 表情 - Map hashTagWordsMaps = new HashMap<>(); // 话题 - Map opinionsWordsMaps = new HashMap<>(); // 评价 + // 地点 + Map placesWordsMaps = new HashMap<>(); + // 表情 + Map emojiWordsMaps = new HashMap<>(); + // 话题 + Map hashTagWordsMaps = new HashMap<>(); + // 评价 + Map opinionsWordsMaps = new HashMap<>(); // List hlKeywordsList = new ArrayList<>(); for (ESMonitorEntity esMonitorEntity : esMonitorEntityList) { List hlKeyWords = esMonitorEntity.getHlKeyWords(); @@ -63,10 +67,12 @@ public class SearchKeywordsCouldService{ CollectionUtils.sortByValueForListSubTopSize(placesWordsMaps, topSize * 2, placesWordsResultMap); //sortByValueForListSubTopSize Long keyWordsStart = System.currentTimeMillis(); - jsonObject.put(ConditionCommon.WORD_CLOUD, CollectionUtils.sortByValueForList(textService.post(keyWordsResultMap, 1), topSize)); + //jsonObject.put(ConditionCommon.WORD_CLOUD, CollectionUtils.sortByValueForList(textService.post(keyWordsResultMap, 1), topSize)); + jsonObject.put(ConditionCommon.WORD_CLOUD,CollectionUtils.sortByValueForList(keyWordsResultMap, topSize)); logger.info("[keysWords Execute Time one] the time used is {} ms", System.currentTimeMillis() - keyWordsStart); Long placesWordsStart = System.currentTimeMillis(); - jsonObject.put(ConditionCommon.PLACE_CLOUD, CollectionUtils.sortByValueForList(textService.post(placesWordsResultMap, 2), topSize)); + // jsonObject.put(ConditionCommon.PLACE_CLOUD, CollectionUtils.sortByValueForList(textService.post(placesWordsResultMap, 2), topSize)); + jsonObject.put(ConditionCommon.PLACE_CLOUD, CollectionUtils.sortByValueForList(placesWordsResultMap, topSize)); logger.info("[placeWords Exceute Time two] the time used is {} ms", System.currentTimeMillis() - placesWordsStart); jsonObject.put(ConditionCommon.HASH_TAG_CLOUD, CollectionUtils.sortByValueForList(hashTagWordsMaps, topSize)); jsonObject.put(ConditionCommon.EMOJI_CLOUD, CollectionUtils.sortByValueForList(emojiWordsMaps, topSize)); diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/UpdateService.java b/cl_search_api/src/main/java/com/bfd/mf/service/UpdateService.java index e540e23..ade1025 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/UpdateService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/UpdateService.java @@ -40,7 +40,8 @@ public class UpdateService extends CrudService currentIndexList = new ArrayList<>(); - if(!("").equals(subjectId) && null != subjectId){ // 如果是专题，去专题的索引查就行 - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111 + // 如果是专题，去专题的索引查就行 + if(!("").equals(subjectId) && null != subjectId){ + cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId; } currentIndexList.add(subjectId); diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/UploadExcelService.java b/cl_search_api/src/main/java/com/bfd/mf/service/UploadExcelService.java index ccf8fb0..7a9c927 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/UploadExcelService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/UploadExcelService.java @@ -6,7 +6,6 @@ import com.bfd.mf.common.util.es.EsUtils2; import com.bfd.mf.common.web.entity.mysql.topic.ParseExcelTask; import com.bfd.mf.common.web.repository.mysql.topic.ParseExcelTaskRepository; import com.bfd.mf.config.BFDApiConfig; -import org.assertj.core.util.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -14,10 +13,16 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import javax.annotation.PostConstruct; -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; import java.math.BigInteger; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; diff --git a/cl_stream_3.1.2.iml b/cl_stream_3.1.2.iml deleted file mode 100644 index f409c0e..0000000 --- a/cl_stream_3.1.2.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/pom.xml b/pom.xml index 441f0de..adbf1d4 100644 --- a/pom.xml +++ b/pom.xml @@ -5,8 +5,8 @@ 4.0.0 com.bfd.mf - cl_stream_3.1.2 - 3.1.2-SNAPSHOT + cl_stream_3.2 + 3.2-SNAPSHOT pom