From 0e254541822597cc6e26704edc9f6913e1f73d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=9C=E9=9D=99?= Date: Wed, 14 Jul 2021 19:15:19 +0800 Subject: [PATCH] =?UTF-8?q?20210714-v1=20=E5=90=88=E5=B9=B6=E5=B4=94?= =?UTF-8?q?=E8=80=81=E5=B8=88=E7=89=88=E6=9C=AC=E7=9A=84=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E8=B7=9F=E9=87=87=E9=9B=86=E5=B9=B3=E5=8F=B0=E7=9A=84=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=20=E5=8F=AF=E9=80=82=E9=85=8D=E4=B8=A4=E4=B8=AA?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 23 ++++---- cl_query_data_job/pom.xml | 4 +- cl_search_api/pom.xml | 8 +-- .../common/service/es/ParseSearchScopeService.java | 15 +++--- .../bfd/mf/common/util/constants/ESConstant.java | 2 + .../web/repository/mysql/base/SiteRepository.java | 11 +--- .../web/vo/view/monitor/ESMonitorBaseEntity.java | 12 ++++- .../java/com/bfd/mf/service/SearchDataService.java | 25 +++++++-- .../bfd/mf/service/SearchKeywordsCouldService.java | 63 ---------------------- cl_stream_3.1.1.iml | 12 +++++ pom.xml | 4 +- 11 files changed, 76 insertions(+), 103 deletions(-) create mode 100644 cl_stream_3.1.1.iml diff --git a/README.md b/README.md index b8b4da6..0b86d1c 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ -崔老师项目版本的代码 -采集平台离线服务和查询接口： - -cl_query_data_job:离线统计、离线拉数据 、欧莱雅离线拉数 - -cl_search_api:查询接口 -这一版的列表页有两个下拉标注选项：分类标签和价值标签 -每条数据的详情页会显示 该数据的分类标签和价值标签 -同时提供分类标签和价值标签的统计结果。 - +采集平台离线服务和查询接口&崔老师项目融合版： +cl_search_api: +1、数据查询接口 +2、数据上传接口 +3、数据分析接口 +4、用户查询接口 +cl_query_data_job: +1、统计服务 +2、离线拉数 +3、欧莱雅拉数 +4、上传服务 +5、任务统计服务 +6、报警服务 diff --git a/cl_query_data_job/pom.xml b/cl_query_data_job/pom.xml index 07bfe79..1fa826d 100644 --- a/cl_query_data_job/pom.xml +++ b/cl_query_data_job/pom.xml @@ -4,9 +4,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - cl_stream_4.0 + cl_stream_3.1.1 com.bfd.mf - 4.0-SNAPSHOT + 3.1.1-SNAPSHOT cl_query_data_job diff --git a/cl_search_api/pom.xml b/cl_search_api/pom.xml index d41be19..d87e69e 100644 --- a/cl_search_api/pom.xml +++ b/cl_search_api/pom.xml @@ -5,15 +5,15 @@ 4.0.0 - cl_stream_4.0 + cl_stream_3.1.1 com.bfd.mf - 4.0-SNAPSHOT + 3.1.1-SNAPSHOT cl_search_api - Search V4.0 API + Search V3.1.1 API cl_search_api - 4.0-SNAPSHOT + 3.1.1-SNAPSHOT diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java index 7b1c33a..356880b 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java @@ -57,15 +57,14 @@ public class ParseSearchScopeService { if(searchType == 0 ){ //0:主贴;1:评论;2:用户 || ES 中 primary=1为主贴 searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY, 1)); } else if(searchType == 1){ - searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY, 0)); - // 之前电商主贴评论是一条数据，因此需要下面这样组装查询 -// searchScopeQuery = QueryBuilders.boolQuery() -// .should(QueryBuilders.termQuery(ESConstant.PRIMARY, 0)) -// .should(QueryBuilders.boolQuery() -// .must(QueryBuilders.termQuery(ESConstant.PRIMARY,1)) -// .must(QueryBuilders.termQuery(ESConstant.DOC_TYPE,ESConstant.ITEM))); + searchScopeQuery = QueryBuilders.boolQuery() + .should(QueryBuilders.termQuery(ESConstant.PRIMARY, 0)) + .should(QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(ESConstant.PRIMARY,1)) + .must(QueryBuilders.termQuery(ESConstant.DOC_TYPE,ESConstant.ITEM))); } else if(searchType == 2){ - searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY, 2)); + searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY, 2)) + .must(QueryBuilders.termsQuery(ESConstant.PAGETYPE,"userInfoPage")); } return searchScopeQuery; } diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java index 7e664c7..8f1333b 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java @@ -288,6 +288,8 @@ public class ESConstant { public static final String DATA_COUNT = "dataCount"; + public static final String PAGETYPE = "pageType"; + /** * 回溯开始时间 */ diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/base/SiteRepository.java b/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/base/SiteRepository.java index 499e5bd..17794d4 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/base/SiteRepository.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/base/SiteRepository.java @@ -14,17 +14,10 @@ public interface SiteRepository extends CrudRepository { @Query(value = "SELECT cid,site_id,site_icon,site_type FROM cl_site WHERE site_id IS NOT NULL AND del = 0", nativeQuery = true) List> findsiteByDel(int del); -// @Query(value = "select cid,site_icon from cl_site WHERE del = 0", nativeQuery = true) -// List> findsiteIconByDel(int del); - - @Query(value = "SELECT cid FROM cl_site WHERE area =?1 ", nativeQuery = true) + @Query(value = "select cid from cl_site WHERE area =?1 ", nativeQuery = true) List findCidsByArea(String area); - -// @Query(value = "select id,site_id,cid from cl_site WHERE del = 0", nativeQuery = true) -// Site findAllSiteIds(); - - @Query(value = "SELECT cid,site_id,site_icon,site_type FROM cl_site WHERE cid = ?1 AND del = 0", nativeQuery = true) + @Query(value = "SELECT cid,site_id,site_icon,site_type FROM cl_site WHERE cid = ?1 AND is_usable =0 AND del = 0", nativeQuery = true) List> findSiteByEnSource(String enSource); } diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorBaseEntity.java b/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorBaseEntity.java index 8d39b6a..e8f867c 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorBaseEntity.java +++ b/cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorBaseEntity.java @@ -67,7 +67,7 @@ public class ESMonitorBaseEntity implements Comparable, Ser private String content; // 内容 private String contentSimHash; //文章SimHash private Integer contentSize; //正文长度 - // 转、赞、评 数字 + // 转、赞、评 数字 private String quoteCount ; private String attitudesCount; private Integer commentsCount = 0; @@ -117,6 +117,16 @@ public class ESMonitorBaseEntity implements Comparable, Ser private String categoryLabel; private String tag; + private String otherSourceJson; + + public String getOtherSourceJson() { + return otherSourceJson; + } + + public void setOtherSourceJson(String otherSourceJson) { + this.otherSourceJson = otherSourceJson; + } + public String getTag() { return tag; } diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java index 84c63c8..6e48eb4 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java @@ -183,8 +183,10 @@ public class SearchDataService extends CrudService> siteMap) throws Exception { @@ -261,7 +263,12 @@ public class SearchDataService extends CrudService()); } + } else if(entry.getKey().equals(ESConstant.ATTITUDES_COUNT)){ + if(entry.getValue().toString().contains("totalCount")){ + JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString()); + jsonObject.put(entry.getKey(),totalCount.get("totalCount")); + } } else { jsonObject.put(entry.getKey(), entry.getValue()); } diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchKeywordsCouldService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchKeywordsCouldService.java index a0aef02..d70e2a8 100644 --- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchKeywordsCouldService.java +++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchKeywordsCouldService.java @@ -25,69 +25,6 @@ public class SearchKeywordsCouldService{ @Autowired private TextService textService; - @Autowired - private SliceScrollUtil sliceScrollUtil; - - /** - * 词云统计接口 - */ -// public JSONObject dataAnalysisCloud(QueryRequest queryRequest){ -// long start = System.currentTimeMillis(); -// JSONObject jsonObject = new JSONObject(); -// /**词云返回个数*/ -// int topSize = queryRequest.getLimit(); -// logger.info("[SearchKeywordsCouldService] dataAnalysisCloud : the top size is:{} ", topSize); -// try { -// List cacheEsMonitorEntityList = sliceScrollUtil.fetchResultSubjectCache(queryRequest,ESConstant.FIELD_CLOUD_ANALYSIS); -// -// Map keyWordsMaps = new HashMap<>(); -// Map placesWordsMaps = new HashMap<>(); // 地点 -// Map emojiWordsMaps = new HashMap<>(); // 表情 -// Map hashTagWordsMaps = new HashMap<>(); // 话题 -// Map opinionsWordsMaps = new HashMap<>(); // 评价 -//// List hlKeywordsList = new ArrayList<>(); -// for (ESMonitorEntity esMonitorEntity : cacheEsMonitorEntityList) { -// List hlKeyWords = esMonitorEntity.getHlKeyWords(); -// getMapCloudKeyWords(placesWordsMaps, esMonitorEntity.getPlaces()); -// // 表情 -// getExpressionMapCloudKeyWords(emojiWordsMaps, esMonitorEntity.getExpression()); -// // 话题 -// getMapCloudKeyWords(hashTagWordsMaps, esMonitorEntity.getHashTag()); -// // 评价 -// getMapCloudKeyWords(opinionsWordsMaps, esMonitorEntity.getOpinions()); -// -// Map finalKeyWordsMaps = keyWordsMaps; -// hlKeyWords.forEach(el -> finalKeyWordsMaps.merge(el, 1, (a, b) -> a + b)); -// Set wordsSets = new HashSet<>(Arrays.asList(StringUtils.split(String.valueOf(hlKeyWords), " "))); -// for (String key : wordsSets) { -// if (TStringUtils.isNotEmpty(key) && key.length() > 1 && !key.equals("null")) { -// // 统计 -// key = key.replace("[","").replace("]",""); -// keyWordsMaps.merge(key, weightValue, (a, b) -> a + b); -// } -// } -// } -// Map keyWordsResultMap = new HashMap<>(); -// Map placesWordsResultMap = new HashMap<>(); -// CollectionUtils.sortByValueForListSubTopSize(keyWordsMaps, topSize * 2, keyWordsResultMap); -// CollectionUtils.sortByValueForListSubTopSize(placesWordsMaps, topSize * 2, placesWordsResultMap); -// //sortByValueForListSubTopSize -// Long keyWordsStart = System.currentTimeMillis(); -// jsonObject.put(ConditionCommon.WORD_CLOUD, CollectionUtils.sortByValueForList(textService.post(keyWordsResultMap, 1), topSize)); -// logger.info("[keysWords Execute Time one] the time used is {} ms", System.currentTimeMillis() - keyWordsStart); -// Long placesWordsStart = System.currentTimeMillis(); -// jsonObject.put(ConditionCommon.PLACE_CLOUD, CollectionUtils.sortByValueForList(textService.post(placesWordsResultMap, 2), topSize)); -// logger.info("[placeWords Exceute Time two] the time used is {} ms", System.currentTimeMillis() - placesWordsStart); -// jsonObject.put(ConditionCommon.HASH_TAG_CLOUD, CollectionUtils.sortByValueForList(hashTagWordsMaps, topSize)); -// jsonObject.put(ConditionCommon.EMOJI_CLOUD, CollectionUtils.sortByValueForList(emojiWordsMaps, topSize)); -// jsonObject.put(ConditionCommon.OPINION_CLOUD, CollectionUtils.sortByValueForList(opinionsWordsMaps, topSize)); -// -// logger.info("[SearchKeywordsCouldService ] the time used is {} ms", (System.currentTimeMillis() - start)); -// } catch (Exception e) { -// logger.error("[SearchKeywordsCouldService] Is Error", e); -// } -// return jsonObject; -// } public JSONObject dataAnalysisCloud( List esMonitorEntityList){ diff --git a/cl_stream_3.1.1.iml b/cl_stream_3.1.1.iml new file mode 100644 index 0000000..f409c0e --- /dev/null +++ b/cl_stream_3.1.1.iml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 7e26df6..fd8edc1 100644 --- a/pom.xml +++ b/pom.xml @@ -5,8 +5,8 @@ 4.0.0 com.bfd.mf - cl_stream_4.0 - 4.0-SNAPSHOT + cl_stream_3.1.1 + 3.1.1-SNAPSHOT pom