From de289cffbc350d9b83663fbcc89b6bf2841c1ce4 Mon Sep 17 00:00:00 2001 From: "jing.du" Date: Tue, 28 Feb 2023 10:31:54 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E4=BA=86=E4=B8=80=E4=BA=9BES=E7=9A=84?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E6=96=B9=E5=BC=8F=EF=BC=8C=E4=B9=8B=E5=89=8D?= =?UTF-8?q?=E6=98=AF=E6=9C=89=E4=B8=A4=E7=A7=8D=EF=BC=8C=E7=8E=B0=E5=9C=A8?= =?UTF-8?q?=E6=94=B9=E6=88=90=E4=B8=80=E7=A7=8D=E4=BA=86=EF=BC=8C=E4=BD=86?= =?UTF-8?q?=E6=98=AF=E6=9C=89=E4=B8=80=E5=9D=97=E6=B6=89=E5=8F=8A=E5=88=B0?= =?UTF-8?q?=E7=9A=84=E5=8F=82=E6=95=B0=E6=AF=94=E8=BE=83=E5=A4=9A=EF=BC=8C?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E5=AE=8C=E5=85=A8=E4=BF=AE=E6=94=B9=EF=BC=8C?= =?UTF-8?q?=E7=AD=89=E6=9C=89=E7=A9=BA=E5=86=8D=E8=B0=83=E6=95=B4=E4=B8=80?= =?UTF-8?q?=E4=B8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cl_search_api/pom.xml | 2 +- .../common/service/es/ParseSearchScopeService.java | 3 +- .../common/service/es/SubjectQueryDataService.java | 22 + .../java/com/bfd/mf/common/util/ESServerUtils.java | 2 +- .../java/com/bfd/mf/common/util/es/EsUtils.java | 11 +- .../bfd/mf/common/util/slice/SliceScrollUtil.java | 15 +- .../com/bfd/mf/service/SearchAuthorService.java | 16 +- .../java/com/bfd/mf/service/SearchDataService.java | 483 ++++++++++----------- .../java/com/bfd/mf/service/UpdateService.java | 6 +- cl_search_api/src/main/resources/application.yml | 37 +- 10 files changed, 315 insertions(+), 282 deletions(-) diff --git a/cl_search_api/pom.xml b/cl_search_api/pom.xml index f0ddb5c..9decd64 100644 --- a/cl_search_api/pom.xml +++ b/cl_search_api/pom.xml @@ -13,7 +13,7 @@ cl_search_api Search V3.2 API cl_search_api - 3.2.4-SNAPSHOT + 3.2.5-SNAPSHOT diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java index 20a92b6..12b8798 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java @@ -55,7 +55,8 @@ public class ParseSearchScopeService { public static BoolQueryBuilder getSearchTypeQuery(Integer searchType) { BoolQueryBuilder searchScopeQuery = null; if(searchType == 0 ){ //0:主贴;1:评论;2:用户 || ES 中 primary=1为主贴 - searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY, 1)); + searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY, + 1)); } else if(searchType == 1){ searchScopeQuery = QueryBuilders.boolQuery() .should(QueryBuilders.termQuery(ESConstant.PRIMARY, 0)) diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java index 3f38359..d56757b 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java @@ -7,6 +7,7 @@ import com.bfd.mf.common.util.constants.ESConstant; import com.bfd.mf.common.util.thread.SubjectDataQueryThread; import com.bfd.mf.common.web.entity.mysql.cache.Cluster; import com.bfd.mf.common.web.vo.params.QueryRequest; +import com.bfd.mf.config.BFDApiConfig; import com.bfd.nlp.common.util.string.TStringUtils; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -38,6 +39,9 @@ public class SubjectQueryDataService { private Logger logger = LoggerFactory.getLogger(SubjectQueryDataService.class); private static final Long ONE_DAY = 60 * 60 * 1000L * 24; + + @Autowired + private BFDApiConfig bfdApiConfig; @Autowired private TopicQueryService topicQueryService; @Autowired @@ -156,4 +160,22 @@ public class SubjectQueryDataService { return new ArrayList<>(set); } + /** + * 现在用这个方法 + * @param subjectIds + * @return + */ + public List getIndexBySubjectIds(String subjectIds) { + Set set = new HashSet<>(); + if(subjectIds.contains(",")){ + for(String subjectId :subjectIds.split(",")){ + set.add(bfdApiConfig.getIndexNamePre() + subjectId); + } + }else { + String subjectId = subjectIds; + set.add(bfdApiConfig.getIndexNamePre() + subjectId); + } + return new ArrayList<>(set); + } + } \ No newline at end of file diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java index 0123d7c..a6d633a 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java @@ -42,7 +42,7 @@ import java.util.Map; */ @Component public class ESServerUtils implements Serializable { - private static Logger logger = LoggerFactory.getLogger(ESServerUtils.class); + private final static Logger logger = LoggerFactory.getLogger(ESServerUtils.class); private static final long serialVersionUID = 1L; private static Map clientByClusterId = new HashMap<>(); @Autowired diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java index 53a46d7..fe5ebb3 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java @@ -241,11 +241,16 @@ public abstract class EsUtils { .setQuery(queryBuilder) .addAggregation(aggregation); - // System.out.println(requestBuilder); + System.out.println(requestBuilder); Aggregations aggregations = requestBuilder.get().getAggregations(); Cardinality cardinality = aggregations.get(count); - // System.out.println("1111 : "+cardinality.getValue()); - return cardinality.getValue(); + System.out.println("1111 : " + cardinality.getValue()); + System.out.println("2222 : " + requestBuilder.get().getHits().totalHits); + long resultCount = cardinality.getValue(); + if(searchType == 2){ + resultCount = requestBuilder.get().getHits().totalHits; + } + return resultCount; } diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java index 87af8d9..1acbf49 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java @@ -1,6 +1,5 @@ package com.bfd.mf.common.util.slice; -import com.bfd.mf.common.service.es.ClusterService; import com.bfd.mf.common.service.es.SubjectQueryDataService; import com.bfd.mf.common.util.constants.ESConstant; import com.bfd.mf.common.web.entity.mysql.cache.Cluster; @@ -25,8 +24,6 @@ import java.util.stream.Collectors; public class SliceScrollUtil { private static final Logger logger = LoggerFactory.getLogger(SliceScrollUtil.class); @Autowired - private ClusterService clusterService; - @Autowired private SubjectQueryDataService subjectQueryDataService; /** @@ -131,12 +128,12 @@ public class SliceScrollUtil { List currentIndexList = null; if(null != queryRequest.getSubjectId() && !("all").equals(queryRequest.getSubjectId())){ logger.info("查询 【专题数据】 subjectId :{}" ,queryRequest.getSubjectId()); - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); - currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); - }else{ - logger.info("[SliceScrollUtil] fetchResultSubjectCache : 查询 【全局数据】"); - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); - currentIndexList = subjectQueryDataService.getIndexListByTimeRange(cluster, queryRequest.getStartTime(),queryRequest.getEndTime()); + //cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); + currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId()); +// }else{ +// logger.info("[SliceScrollUtil] fetchResultSubjectCache : 查询 【全局数据】"); +// cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); +// currentIndexList = subjectQueryDataService.getIndexListByTimeRange(queryRequest.getStartTime(),queryRequest.getEndTime()); } Long clusterId = cluster.getId(); diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java index 107d181..ccb2366 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java @@ -2,7 +2,6 @@ package com.bfd.mf.service; import com.alibaba.fastjson.JSONObject; import com.bfd.mf.common.service.common.CrudService; -import com.bfd.mf.common.service.es.ClusterService; import com.bfd.mf.common.service.es.EsQueryAuthorCountService; import com.bfd.mf.common.service.es.EsQueryAuthorService; import com.bfd.mf.common.service.es.SubjectQueryDataService; @@ -13,6 +12,7 @@ import com.bfd.mf.common.web.repository.mysql.SentimentRepository; import com.bfd.mf.common.web.repository.mysql.base.SiteRepository; import com.bfd.mf.common.web.vo.params.QueryRequest; import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity; +import com.bfd.mf.config.BFDApiConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -28,7 +28,7 @@ import java.util.Map; public class SearchAuthorService extends CrudService implements Serializable { private static Logger logger = LoggerFactory.getLogger(SearchAuthorService.class); @Autowired - private ClusterService clusterService; + private BFDApiConfig bfdApiConfig; @Autowired private SubjectQueryDataService subjectQueryDataService; @Autowired @@ -75,8 +75,8 @@ public class SearchAuthorService extends CrudService dataList = esQueryAuthorService.queryAuthorByAuthorId(indexNames, queryRequest); jsonObject = parseAuthorMessage(dataList); @@ -135,8 +135,8 @@ public class SearchAuthorService extends CrudService dataList = esQueryAuthorService.queryContentsByAuthorId(indexNames, queryRequest); List esMonitorEntityLists = new ArrayList<>(); @@ -280,8 +280,8 @@ public class SearchAuthorService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); + // Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111 + List currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId()); String[] indexName = currentIndexList.toArray(new String[currentIndexList.size()]); return indexName; } diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java index 3e726b6..0ae01a7 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java @@ -37,8 +37,6 @@ public class SearchDataService extends CrudService dataIdList, - String orderFlag,String sortFlag, + private SearchResponse buildDataIdQueryCrawl(Integer from, Integer searchSize, List dataIdList, + String orderFlag, String sortFlag, List currentIndexList, Cluster cluster) { - if(sortFlag.equals(ESConstant.COMMENT)){ + if (sortFlag.equals(ESConstant.COMMENT)) { sortFlag = ESConstant.COMMENTS_COUNT; } - if(sortFlag.equals("")){ + if (sortFlag.equals("")) { sortFlag = ESConstant.PUBTIME; } QueryBuilder queryBuilder = esCommonService.buildStringQueryByField(ESConstant.DATA_ID, dataIdList); @@ -88,19 +86,20 @@ public class SearchDataService extends CrudService dataList, List esMonitorListEntity,Integer searchType) { + private void parseQueryResult(List dataList, List esMonitorListEntity, Integer searchType) { try { List> site = siteRepository.findsiteByDel(0); - Map> siteMap = new HashMap<>(); + Map> siteMap = new HashMap<>(); for (Map map : site) { - siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(),map); + siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(), map); } if (null != dataList && dataList.size() > 0) { for (JSONObject json : dataList) { @@ -108,7 +107,7 @@ public class SearchDataService extends CrudService sourceAsMap = firstHit.getSourceAsMap(); String title = "标题为空"; - if(sourceAsMap.containsKey(ESConstant.TITLE)) { + if (sourceAsMap.containsKey(ESConstant.TITLE)) { title = sourceAsMap.get(ESConstant.TITLE).toString(); } String content = sourceAsMap.get(ESConstant.CONTENT).toString(); @@ -130,12 +129,12 @@ public class SearchDataService extends CrudService> siteMap) throws Exception { + private ESMonitorEntity parseMainMessage(JSONObject jsonObject, Integer searchType, + Map> siteMap) throws Exception { ESMonitorEntity esMonitorEntity = new ESMonitorEntity(); try { Map sourceAsMap = jsonObject; @@ -215,9 +214,9 @@ public class SearchDataService extends CrudService siteOtherMap = siteMap.get(enSource); if (siteOtherMap.containsKey("site_id")) { siteId = siteMap.get(enSource).get("site_id").toString(); @@ -255,10 +254,10 @@ public class SearchDataService extends CrudService hlKeywords = new ArrayList<>(); - if (sourceAsMap.get(ESConstant.HL_KEYWORDS) instanceof List){ + if (sourceAsMap.get(ESConstant.HL_KEYWORDS) instanceof List) { hlKeywords = (List) sourceAsMap.get(ESConstant.HL_KEYWORDS); } //List // 视频分析结果 String asrText = ""; List ocrText = new ArrayList<>(); - if(sourceAsMap.containsKey(ESConstant.ASRTEXT)) { + if (sourceAsMap.containsKey(ESConstant.ASRTEXT)) { asrText = sourceAsMap.get(ESConstant.ASRTEXT).toString(); } - if(sourceAsMap.containsKey(ESConstant.OCRTEXT)) { + if (sourceAsMap.containsKey(ESConstant.OCRTEXT)) { ocrText = (List) sourceAsMap.get(ESConstant.OCRTEXT); } // 如果是用户数据，需要获取下面四个字段值 @@ -409,7 +408,7 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>(); String subjectId = queryRequest.getSubjectId(); - if(null != queryRequest.getSubjectId() && !("").equals(subjectId)){ // 如果是专题，去专题的索引查就行 - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 - subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId; + if (null != queryRequest.getSubjectId() && !("").equals(subjectId)) { // 如果是专题，去专题的索引查就行 + subjectId =bfdApiConfig.getIndexNamePre() + subjectId; currentIndexList.add(subjectId); - }else{ // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来 + } else { // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来 logger.info("[SearchDataService] queryComment: 查询 全局数据"); - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109 String docId = queryRequest.getDocId(); - String indexType = "cl_index_"+docId.split("_")[1]; + String indexType = "cl_index_" + docId.split("_")[1]; currentIndexList.add(indexType); } - JSONObject result = getCommentListByDocId(queryRequest, cluster,currentIndexList); + JSONObject result = getCommentListByDocId(queryRequest, cluster, currentIndexList); return result; } private JSONObject getCommentListByDocId(QueryRequest queryRequest, Cluster cluster, List currentIndexList) { JSONObject json = new JSONObject(); - try{ + try { String docId = queryRequest.getDocId(); /**依据文档Id查询对应的文档*/ QueryBuilder queryBuilder = esCommonService.buildKeyWordsQueryBuilder(docId, ESConstant.DOC_ID); @@ -621,28 +618,28 @@ public class SearchDataService extends CrudService comments = new ArrayList<>(); Long size = 0L; - if(null != queryRequest.getDataId() && !queryRequest.getDataId().equals("")){ + if (null != queryRequest.getDataId() && !queryRequest.getDataId().equals("")) { String dataId = queryRequest.getDataId(); // String docType = queryRequest.getDocType(); - JSONObject TopComment = getCommentByDataId(cluster,currentIndexList,dataId,docType); + JSONObject TopComment = getCommentByDataId(cluster, currentIndexList, dataId, docType); // System.out.println(TopComment); - if(TopComment.size() > 0) { + if (TopComment.size() > 0) { size = 1L; comments.add(TopComment); } - boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.DATA_ID,dataId)); + boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.DATA_ID, dataId)); } String siteId = queryRequest.getSiteId(); @@ -653,7 +650,7 @@ public class SearchDataService extends CrudService result = response[i].getSourceAsMap(); jsonObject.putAll(result); - jsonObject.put(ESConstant.SITEID,siteId); + jsonObject.put(ESConstant.SITEID, siteId); comments.add(jsonObject); } - json.put(ESConstant.COMMENTLISTS,comments); + json.put(ESConstant.COMMENTLISTS, comments); size = size + searchResponse.getHits().getTotalHits(); - json.put(ESConstant.ALLDOCNUMBER,size); - }catch (Exception e){ + json.put(ESConstant.ALLDOCNUMBER, size); + } catch (Exception e) { e.printStackTrace(); } return json; } - private JSONObject getCommentByDataId(Cluster cluster, List currentIndexList, String dataId,String docType) { + private JSONObject getCommentByDataId(Cluster cluster, List currentIndexList, String dataId, String docType) { JSONObject jsonObject = new JSONObject(); - try{ - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.DATA_ID,dataId)); + try { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.DATA_ID, dataId)); // 由于电商的数据的primary =1 因此不加这个条件了, // if(docType.equals(ESConstant.ITEM)){ // boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.PRIMARY, 1)); @@ -692,19 +689,20 @@ public class SearchDataService extends CrudService 0){ + if (response.length > 0) { Map result = response[0].getSourceAsMap(); jsonObject.putAll(result); } - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } return jsonObject; } /** - * 根据docId 查询一条数据的详情及评论列表 + * 根据docId 查询一条数据的详情及评论列表 + * * @param queryRequest * @return */ @@ -714,87 +712,83 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>(); String subjectId = queryRequest.getSubjectId(); - if(!("").equals(subjectId) && null != subjectId){ // 如果是专题，去专题的索引查就行 - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 - subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId; - }else{ // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来 - logger.info("[SearchDataService] queryOneDataByDocId 查询 全局数据 : {}" , subjectId); - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109 + if (!("").equals(subjectId) && null != subjectId) { // 如果是专题，去专题的索引查就行 + subjectId = bfdApiConfig.getIndexNamePre() + subjectId; + } else { // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来 + logger.info("[SearchDataService] queryOneDataByDocId 查询 全局数据 : {}", subjectId); } currentIndexList.add(subjectId); String docId = queryRequest.getDocId(); // 根据ID 获取一条详情数据 - JSONObject jsonObject = getOneDataByDocId(docId, cluster, currentIndexList); + JSONObject jsonObject = getOneDataByDocId(docId, cluster, currentIndexList); jsonObject = setLable(jsonObject); // 替换几个 pathSize 中的链接的前缀 - if(jsonObject.containsKey(ESConstant.IMAGEPATHSIZE)){ - List>imagePathSize = (List>) jsonObject.get(ESConstant.IMAGEPATHSIZE); - for (Map imagePath: imagePathSize) { + if (jsonObject.containsKey(ESConstant.IMAGEPATHSIZE)) { + List> imagePathSize = (List>) jsonObject.get(ESConstant.IMAGEPATHSIZE); + for (Map imagePath : imagePathSize) { String url = imagePath.get(ESConstant.URL); - url = url.replace(bfdApiConfig.getGoFastDomain(),"").replace("http://172.18.1.113:8892",""); - imagePath.put(ESConstant.URL,url); + url = url.replace(bfdApiConfig.getGoFastDomain(), "").replace("http://172.18.1.113:8892", ""); + imagePath.put(ESConstant.URL, url); } } - if(jsonObject.containsKey(ESConstant.VIDEOPATHSIZE)){ - List>videoPathSize = (List>) jsonObject.get(ESConstant.VIDEOPATHSIZE); - for (Map videoPath: videoPathSize) { + if (jsonObject.containsKey(ESConstant.VIDEOPATHSIZE)) { + List> videoPathSize = (List>) jsonObject.get(ESConstant.VIDEOPATHSIZE); + for (Map videoPath : videoPathSize) { String url = videoPath.get(ESConstant.URL); - url = url.replace(bfdApiConfig.getGoFastDomain(),"").replace("http://172.18.1.113:8892",""); - videoPath.put(ESConstant.URL,url); + url = url.replace(bfdApiConfig.getGoFastDomain(), "").replace("http://172.18.1.113:8892", ""); + videoPath.put(ESConstant.URL, url); } } - if(jsonObject.containsKey(ESConstant.FILEPATHSIZE)){ //http://172.18.1.113:8080 - List>filePathSize = (List>) jsonObject.get(ESConstant.FILEPATHSIZE); - for (Map filePath: filePathSize) { + if (jsonObject.containsKey(ESConstant.FILEPATHSIZE)) { //http://172.18.1.113:8080 + List> filePathSize = (List>) jsonObject.get(ESConstant.FILEPATHSIZE); + for (Map filePath : filePathSize) { String url = filePath.get(ESConstant.URL); - url = url.replace(bfdApiConfig.getGoFastDomain(),"").replace("http://172.18.1.113:8892",""); - filePath.put(ESConstant.URL,url); + url = url.replace(bfdApiConfig.getGoFastDomain(), "").replace("http://172.18.1.113:8892", ""); + filePath.put(ESConstant.URL, url); } } String enSource = jsonObject.getString(ESConstant.EN_SOURCE); - if(null != enSource) { + if (null != enSource) { jsonObject = getSite(jsonObject, enSource); } // 如果是社交媒体类的数据，需要将 author 放到 title 字段中做显示 jsonObject = socialDataChangeAuthorAndTitle(jsonObject); // 根据用户ID 查询一下用户信息，如果没有的话，就把原来的用户名放进去。 String docType = jsonObject.getString(ESConstant.DOC_TYPE); - jsonObject = getUserMessageByAuthorId(jsonObject,docType,queryRequest,subjectId); + jsonObject = getUserMessageByAuthorId(jsonObject, docType, queryRequest, subjectId); // 如果是电商数据，需要把电商详情信息填到 content 中，并把一些店铺信息填到 author中 - jsonObject = getItemDataShopMeggage(jsonObject,docType); + jsonObject = getItemDataShopMeggage(jsonObject, docType); return jsonObject; } - - // 崔老师那个版本，需要添加 分类标签和 价值标签 private JSONObject setLable(JSONObject jsonObject) { - if(!jsonObject.containsKey(ESConstant.VALUELABEL) || null == jsonObject.get(ESConstant.VALUELABEL)){ - jsonObject.put(ESConstant.VALUELABEL,""); - }else{ - jsonObject.put(ESConstant.VALUELABEL,jsonObject.get(ESConstant.VALUELABEL).toString()); + if (!jsonObject.containsKey(ESConstant.VALUELABEL) || null == jsonObject.get(ESConstant.VALUELABEL)) { + jsonObject.put(ESConstant.VALUELABEL, ""); + } else { + jsonObject.put(ESConstant.VALUELABEL, jsonObject.get(ESConstant.VALUELABEL).toString()); } - if(!jsonObject.containsKey(ESConstant.CATEGORYLABEL)){ - jsonObject.put(ESConstant.CATEGORYLABEL,""); + if (!jsonObject.containsKey(ESConstant.CATEGORYLABEL)) { + jsonObject.put(ESConstant.CATEGORYLABEL, ""); } return jsonObject; } private JSONObject getSite(JSONObject jsonObject, String enSource) { List> site = siteRepository.findSiteByEnSource(enSource); - Map> siteMap = new HashMap<>(); + Map> siteMap = new HashMap<>(); for (Map map : site) { - siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(),map); + siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(), map); } String siteId = ""; String icon = ""; String siteType = ""; - Map siteOtherMap = siteMap.get(enSource); - if(enSource.equals(ESConstant.SINA)){ + Map siteOtherMap = siteMap.get(enSource); + if (enSource.equals(ESConstant.SINA)) { siteId = "183"; - }else { + } else { if (siteOtherMap.containsKey("site_id")) { siteId = siteMap.get(enSource).get("site_id").toString(); } @@ -805,21 +799,21 @@ public class SearchDataService extends CrudService0) { + System.out.println("本次查询结果条数：" + searchDataResponse.getHits().totalHits); + if (searchDataResponse.getHits().getHits().length > 0) { Map result = searchDataResponse.getHits().getHits()[0].getSourceAsMap(); - if (result.size() > 0) { result.entrySet().stream() - .forEach(entry -> { - if (entry.getKey().equals(ESConstant.FILEPATH) && entry.getValue().equals("")) { - jsonObject.put(entry.getKey(), new ArrayList<>()); - } else if (entry.getKey().equals(ESConstant.FILEPATHSIZE) - || entry.getKey().equals(ESConstant.IMAGEPATHSIZE) - || entry.getKey().equals(ESConstant.VIDEOPATHSIZE)) { - if(entry.getValue().toString() .contains(ESConstant.URL)) { + if (result.size() > 0) { + result.entrySet().stream() + .forEach(entry -> { + if (entry.getKey().equals(ESConstant.FILEPATH) && entry.getValue().equals("")) { + jsonObject.put(entry.getKey(), new ArrayList<>()); + } else if (entry.getKey().equals(ESConstant.FILEPATHSIZE) + || entry.getKey().equals(ESConstant.IMAGEPATHSIZE) + || entry.getKey().equals(ESConstant.VIDEOPATHSIZE)) { + if (entry.getValue().toString().contains(ESConstant.URL)) { + jsonObject.put(entry.getKey(), entry.getValue()); + } else { + jsonObject.put(entry.getKey(), new ArrayList<>()); + } + } else if (entry.getKey().equals(ESConstant.ATTITUDES_COUNT)) { + if (entry.getValue().toString().contains(ESConstant.TOTALCOUNT)) { + JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString()); + jsonObject.put(entry.getKey(), totalCount.get(ESConstant.TOTALCOUNT)); + } else { + jsonObject.put(entry.getKey(), entry.getValue()); + } + } else { jsonObject.put(entry.getKey(), entry.getValue()); - }else{ - jsonObject.put(entry.getKey(),new ArrayList<>()); } - } else if(entry.getKey().equals(ESConstant.ATTITUDES_COUNT)){ - if(entry.getValue().toString().contains(ESConstant.TOTALCOUNT)){ - JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString()); - jsonObject.put(entry.getKey(),totalCount.get(ESConstant.TOTALCOUNT)); - }else{ - jsonObject.put(entry.getKey(),entry.getValue()); - } - } else { - jsonObject.put(entry.getKey(), entry.getValue()); - } - }); + }); } } return jsonObject; @@ -912,10 +907,10 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>(); // 获取ES的参数及要查询的索引列表 String subjectId = queryRequest.getSubjectId(); - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 special_cluster_type - currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster,subjectId); + // cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 special_cluster_type + currentIndexList = subjectQueryDataService.getIndexBySubjectIds(subjectId); Long clusterId = cluster.getId(); - logger.info("[SearchDataService] queryDataList clusterId : {}; currentIndexList : {}" ,clusterId, currentIndexList.toString()); + logger.info("[SearchDataService] queryDataList clusterId : {}; currentIndexList : {}", clusterId, currentIndexList.toString()); String orderFlag = queryRequest.getOrder(); // 排序方式 asc/desc String sortFlag = queryRequest.getSidx(); // 排序字段 @@ -927,15 +922,15 @@ public class SearchDataService extends CrudService dataIdList = new ArrayList<>(); List cacheList = subjectQueryDataService.fetchResponseDataFromCache( - queryRequest,clusterId, + queryRequest, clusterId, orderFlag, sortFlag, currentIndexList, ESConstant.FIELD_ID_LIST); - if(cacheList.size() == 0){ + if (cacheList.size() == 0) { logger.info("没有查到相关数据哦！"); - jsonObject.put(ESConstant.ALLDOCNUMBER,foldDocAllNumber);// 实际查询总量 - jsonObject.put(ESConstant.MONITORLISTS,esMonitorEntityLists); + jsonObject.put(ESConstant.ALLDOCNUMBER, foldDocAllNumber);// 实际查询总量 + jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists); return jsonObject; } @@ -949,13 +944,13 @@ public class SearchDataService extends CrudService timeSeries = new ArrayList<>(values); Integer limit = queryRequest.getLimit(); //每页的数量 - Integer start = (queryRequest.getPage()-1)*limit; //起始页(0,20,40....) + Integer start = (queryRequest.getPage() - 1) * limit; //起始页(0,20,40....) // 将查询结果的 dataId 写入到 dataIdList中 - Map dedupmap = new HashMap<>(); - if(timeSeries.size() > 0) { + Map dedupmap = new HashMap<>(); + if (timeSeries.size() > 0) { for (int i = start; i < timeSeries.size(); i++) { - if(null != timeSeries.get(i).getDataId() && !("").equals(timeSeries.get(i).getDataId())) { + if (null != timeSeries.get(i).getDataId() && !("").equals(timeSeries.get(i).getDataId())) { dedupmap.put(timeSeries.get(i).getDataId(), timeSeries.get(i).getDataId()); } } @@ -972,7 +967,7 @@ public class SearchDataService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); - Long clusterId = cluster.getId(); - logger.info("[SearchDataService] queryDataInOneIndex: clusterId :{} ; currentIndexList : {}", clusterId,currentIndexList.toString()); + // Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); + List currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId()); + //Long clusterId = cluster.getId(); + logger.info("[SearchDataService] queryDataInOneIndex: currentIndexList : {}", currentIndexList.toString()); // String indexName = currentIndexList.get(0); - String indexNames [] = currentIndexList.toArray(new String [currentIndexList.size()]); + String indexNames[] = currentIndexList.toArray(new String[currentIndexList.size()]); List dataList = esQueryServiceForSQMini.queryDataFromOneSubject(indexNames, queryRequest); List esMonitorEntityLists = new ArrayList<>(); Integer searchType = queryRequest.getSearchType(); parseQueryResult(dataList, esMonitorEntityLists, searchType); - Long totalCount = esQueryServiceForSQMini.queryDataCountFromOneSubject(indexNames,queryRequest); - logger.info("[SearchDataService] queryDataInOneIndex: {}",totalCount); - jsonObject.put(ESConstant.ALLDOCNUMBER,totalCount); - jsonObject.put(ESConstant.MONITORLISTS,esMonitorEntityLists); + Long totalCount = esQueryServiceForSQMini.queryDataCountFromOneSubject(indexNames, queryRequest); + logger.info("[SearchDataService] queryDataInOneIndex: {}", totalCount); + jsonObject.put(ESConstant.ALLDOCNUMBER, totalCount); + jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists); - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } return jsonObject; } /** - * 导出专题下数据调用的方法 + * 导出专题下数据调用的方法 + * * @param queryRequest * @return */ @@ -1025,38 +1021,38 @@ public class SearchDataService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); - Long clusterId = cluster.getId(); - String [] indexName = currentIndexList.toArray(new String[currentIndexList.size()]); - logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; clusterId :{} ; currentIndexList :{}",indexName[0], clusterId , currentIndexList.toString()); + //Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); + List currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId()); + // Long clusterId = cluster.getId(); + String[] indexName = currentIndexList.toArray(new String[currentIndexList.size()]); + logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; currentIndexList :{}", indexName[0], currentIndexList.toString()); // 开始查询 - jsonObject= esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest); + jsonObject = esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest); List dataList = (List) jsonObject.get(ESConstant.MONITORLISTS); List esMonitorEntityLists = new ArrayList<>(); Integer searchType = queryRequest.getSearchType(); - parseQueryResult(dataList, esMonitorEntityLists,searchType); + parseQueryResult(dataList, esMonitorEntityLists, searchType); jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists); - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } return jsonObject; } - public JSONObject exportJsonDataInSubject(QueryRequest queryRequest){ + public JSONObject exportJsonDataInSubject(QueryRequest queryRequest) { JSONObject jsonObject = new JSONObject(); - try{ + try { // 获取 ES 的连接方式及要查询的索引列表 - Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 专题索引 special_cluster_type - List currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); - Long clusterId = cluster.getId(); - String [] indexName = currentIndexList.toArray(new String[currentIndexList.size()]); - logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; clusterId :{} ; currentIndexList :{}",indexName[0], clusterId , currentIndexList.toString()); - jsonObject= esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest); - }catch (Exception e){ + // Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 专题索引 special_cluster_type + List currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId()); + // Long clusterId = cluster.getId(); + String[] indexName = currentIndexList.toArray(new String[currentIndexList.size()]); + logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; currentIndexList :{}", indexName[0], currentIndexList.toString()); + jsonObject = esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest); + } catch (Exception e) { e.printStackTrace(); } - return jsonObject; + return jsonObject; } public JSONObject exportDataInDateIndex(QueryRequest queryRequest) { @@ -1066,23 +1062,23 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>(); // 获取ES的参数及要查询的索引列表 String subjectId = queryRequest.getSubjectId(); - if(subjectId.equals("all")) { - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109 - currentIndexList = subjectQueryDataService.getIndexListByTimeRange(cluster, queryRequest.getStartTime(), queryRequest.getEndTime()); - }else if(subjectId.contains(",")){ - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 - currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); + if (subjectId.equals("all")) { + // cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109 + // currentIndexList = subjectQueryDataService.getIndexListByTimeRange(queryRequest.getStartTime(), queryRequest.getEndTime()); + } else if (subjectId.contains(",")) { + // cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 + currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId()); } Long clusterId = cluster.getId(); - logger.info("[SearchDataService] exportDataInOneIndex: clusterId :{}; currentIndexList : {}" , clusterId , currentIndexList.toString()); + logger.info("[SearchDataService] exportDataInOneIndex: clusterId :{}; currentIndexList : {}", clusterId, currentIndexList.toString()); // 开始查询 - jsonObject= esQueryServiceForSQNormal.exportDataFromIndexs(currentIndexList, queryRequest); + jsonObject = esQueryServiceForSQNormal.exportDataFromIndexs(currentIndexList, queryRequest); List dataList = (List) jsonObject.get(ESConstant.MONITORLISTS); List esMonitorEntityLists = new ArrayList<>(); Integer searchType = queryRequest.getSearchType(); - parseQueryResult(dataList, esMonitorEntityLists,searchType); - jsonObject.put(ESConstant.MONITORLISTS,esMonitorEntityLists); - }catch (Exception e){ + parseQueryResult(dataList, esMonitorEntityLists, searchType); + jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists); + } catch (Exception e) { e.printStackTrace(); } return jsonObject; @@ -1090,13 +1086,10 @@ public class SearchDataService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId()); - Long clusterId = cluster.getId(); - logger.info("[SearchDataService] queryDataCountsInOneIndex: clusterId : {}; currentIndexList : {}" , clusterId ,currentIndexList.toString()); - // String indexName = currentIndexList.get(0); - String indexNames [] = currentIndexList.toArray(new String [currentIndexList.size()]); + try { + String subjectId = queryRequest.getSubjectId(); + String indexName = bfdApiConfig.getIndexNamePre()+subjectId; + String indexNames[] = {indexName}; Long contentCount = 0L; Long commentCount = 0L; Long authorCount = 0L; @@ -1107,13 +1100,13 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>(); // 如果是专题，去专题的索引查就行 if(!("").equals(subjectId) && null != subjectId){ - cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); - subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId; + subjectId = bfdApiConfig.getIndexNamePre()+ subjectId; } currentIndexList.add(subjectId); Map params = new HashMap<>(16); diff --git a/cl_search_api/src/main/resources/application.yml b/cl_search_api/src/main/resources/application.yml index 58a1028..82dacbc 100644 --- a/cl_search_api/src/main/resources/application.yml +++ b/cl_search_api/src/main/resources/application.yml @@ -1,5 +1,5 @@ server: - port: 18909 + port: 18902 tomcat: uri-encoding: UTF-8 max-threads: 800 @@ -15,12 +15,16 @@ server: spring: datasource: driver-class-name: com.mysql.cj.jdbc.Driver - username: crawl - password: crawl123 - url: jdbc:mysql://172.18.1.134:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC +# username: crawl +# password: crawl123 +# url: jdbc:mysql://172.18.1.134:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC # username: root # password: bfd123 # url: jdbc:mysql://172.26.11.113:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC + username: crawl + password: crawl123 + url: jdbc:mysql://172.26.11.110:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC + hikari: maximum-pool-size: 10 minimum-idle: 1 @@ -46,16 +50,31 @@ bfd.api.mf: uploadZipPath : /opt/nfsdata/uploadFiles/ indexNamePre : cl_major_ - es-mini: - name: SQ_Mini_2 - address: 172.18.1.81:9301 - upper: 2018-09-01 - standby: cl_major_* +# es-mini: +# name: CL_Mini_2 +# address: 172.18.1.81:9301 +# upper: 2018-09-01 +# standby: cl_major_* es-normal: name: SQ_Normal_new address: 172.18.1.134:9301 upper: 2018-09-01 standby: cl_index_* +# es-mini: #这个是自己部署的6.8.23 的ES +# name: my-application +# address: 172.16.10.39:9300 +# upper: 2018-09-01 +# standby: cl_major_* +# es-mini: # 这个是庆鹏那边搞的 6.8.13 且需要用户验证的 +# name: elasticsearch +# address: 172.16.10.42:9300 +# upper: 2018-09-01 +# standby: cl_major_* + es-mini: # 这个是28赛博项目阿里云上的ES + name: crawl + address: 47.254.148.208:9300 + upper: 2018-09-01 + standby: cl_major_* # es-mini: