zhicheng
/
swcl_search_api
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

改了一些ES的链接方式，之前是有两种，现在改成一种了，但是有一块涉及到的参数比较多，没有完全修改，等有空再调整一下

release-1.0
jing.du 3 years ago
parent
ea589f7e83
commit
de289cffbc
10 changed files with 315 additions and 282 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      cl_search_api/pom.xml
  2. 3
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java
  3. 22
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java
  4. 2
      cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java
  5. 11
      cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java
  6. 15
      cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java
  7. 16
      cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java
  8. 443
      cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java
  9. 6
      cl_search_api/src/main/java/com/bfd/mf/service/UpdateService.java
  10. 37
      cl_search_api/src/main/resources/application.yml

2
cl_search_api/pom.xml
View File

@ -13,7 +13,7 @@
<name>cl_search_api</name> <name>cl_search_api</name>
<description>Search V3.2 API</description> <description>Search V3.2 API</description>
<artifactId>cl_search_api</artifactId> <artifactId>cl_search_api</artifactId>
<version>3.2.4-SNAPSHOT</version>
<version>3.2.5-SNAPSHOT</version>

3
cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java
View File

@ -55,7 +55,8 @@ public class ParseSearchScopeService {
public static BoolQueryBuilder getSearchTypeQuery(Integer searchType) { public static BoolQueryBuilder getSearchTypeQuery(Integer searchType) {
BoolQueryBuilder searchScopeQuery = null; BoolQueryBuilder searchScopeQuery = null;
if(searchType == 0 ){ //0:主贴;1:评论;2:用户 || ES 中 primary=1为主贴 if(searchType == 0 ){ //0:主贴;1:评论;2:用户 || ES 中 primary=1为主贴
searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY, 1));
searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY,
1));
} else if(searchType == 1){ } else if(searchType == 1){
searchScopeQuery = QueryBuilders.boolQuery() searchScopeQuery = QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery(ESConstant.PRIMARY, 0)) .should(QueryBuilders.termQuery(ESConstant.PRIMARY, 0))

22
cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java
View File

@ -7,6 +7,7 @@ import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.util.thread.SubjectDataQueryThread; import com.bfd.mf.common.util.thread.SubjectDataQueryThread;
import com.bfd.mf.common.web.entity.mysql.cache.Cluster; import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
import com.bfd.mf.common.web.vo.params.QueryRequest; import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.config.BFDApiConfig;
import com.bfd.nlp.common.util.string.TStringUtils; import com.bfd.nlp.common.util.string.TStringUtils;
import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
@ -38,6 +39,9 @@ public class SubjectQueryDataService {
private Logger logger = LoggerFactory.getLogger(SubjectQueryDataService.class); private Logger logger = LoggerFactory.getLogger(SubjectQueryDataService.class);
private static final Long ONE_DAY = 60 * 60 * 1000L * 24; private static final Long ONE_DAY = 60 * 60 * 1000L * 24;
@Autowired
private BFDApiConfig bfdApiConfig;
@Autowired @Autowired
private TopicQueryService topicQueryService; private TopicQueryService topicQueryService;
@Autowired @Autowired
@ -156,4 +160,22 @@ public class SubjectQueryDataService {
return new ArrayList<>(set); return new ArrayList<>(set);
} }
/**
* 现在用这个方法
* @param subjectIds
* @return
*/
public List<String> getIndexBySubjectIds(String subjectIds) {
Set<String> set = new HashSet<>();
if(subjectIds.contains(",")){
for(String subjectId :subjectIds.split(",")){
set.add(bfdApiConfig.getIndexNamePre() + subjectId);
}
}else {
String subjectId = subjectIds;
set.add(bfdApiConfig.getIndexNamePre() + subjectId);
}
return new ArrayList<>(set);
}
} }

2
cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java
View File

@ -42,7 +42,7 @@ import java.util.Map;
*/ */
@Component @Component
public class ESServerUtils implements Serializable { public class ESServerUtils implements Serializable {
private static Logger logger = LoggerFactory.getLogger(ESServerUtils.class);
private final static Logger logger = LoggerFactory.getLogger(ESServerUtils.class);
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private static Map<Long, TransportClient> clientByClusterId = new HashMap<>(); private static Map<Long, TransportClient> clientByClusterId = new HashMap<>();
@Autowired @Autowired

11
cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java
View File

@ -241,11 +241,16 @@ public abstract class EsUtils {
.setQuery(queryBuilder) .setQuery(queryBuilder)
.addAggregation(aggregation); .addAggregation(aggregation);
// System.out.println(requestBuilder);
System.out.println(requestBuilder);
Aggregations aggregations = requestBuilder.get().getAggregations(); Aggregations aggregations = requestBuilder.get().getAggregations();
Cardinality cardinality = aggregations.get(count); Cardinality cardinality = aggregations.get(count);
// System.out.println("1111 : "+cardinality.getValue());
return cardinality.getValue();
System.out.println("1111 : " + cardinality.getValue());
System.out.println("2222 : " + requestBuilder.get().getHits().totalHits);
long resultCount = cardinality.getValue();
if(searchType == 2){
resultCount = requestBuilder.get().getHits().totalHits;
}
return resultCount;
} }

15
cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java
View File

@ -1,6 +1,5 @@
package com.bfd.mf.common.util.slice; package com.bfd.mf.common.util.slice;
import com.bfd.mf.common.service.es.ClusterService;
import com.bfd.mf.common.service.es.SubjectQueryDataService; import com.bfd.mf.common.service.es.SubjectQueryDataService;
import com.bfd.mf.common.util.constants.ESConstant; import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.web.entity.mysql.cache.Cluster; import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
@ -25,8 +24,6 @@ import java.util.stream.Collectors;
public class SliceScrollUtil { public class SliceScrollUtil {
private static final Logger logger = LoggerFactory.getLogger(SliceScrollUtil.class); private static final Logger logger = LoggerFactory.getLogger(SliceScrollUtil.class);
@Autowired @Autowired
private ClusterService clusterService;
@Autowired
private SubjectQueryDataService subjectQueryDataService; private SubjectQueryDataService subjectQueryDataService;
/** /**
@ -131,12 +128,12 @@ public class SliceScrollUtil {
List<String> currentIndexList = null; List<String> currentIndexList = null;
if(null != queryRequest.getSubjectId() && !("all").equals(queryRequest.getSubjectId())){ if(null != queryRequest.getSubjectId() && !("all").equals(queryRequest.getSubjectId())){
logger.info("查询 【专题数据】 subjectId :{}" ,queryRequest.getSubjectId()); logger.info("查询 【专题数据】 subjectId :{}" ,queryRequest.getSubjectId());
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
}else{
logger.info("[SliceScrollUtil] fetchResultSubjectCache : 查询 【全局数据】");
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
currentIndexList = subjectQueryDataService.getIndexListByTimeRange(cluster, queryRequest.getStartTime(),queryRequest.getEndTime());
//cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
// }else{
// logger.info("[SliceScrollUtil] fetchResultSubjectCache : 查询 【全局数据】");
// cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
// currentIndexList = subjectQueryDataService.getIndexListByTimeRange(queryRequest.getStartTime(),queryRequest.getEndTime());
} }
Long clusterId = cluster.getId(); Long clusterId = cluster.getId();

16
cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java
View File

@ -2,7 +2,6 @@ package com.bfd.mf.service;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.service.common.CrudService; import com.bfd.mf.common.service.common.CrudService;
import com.bfd.mf.common.service.es.ClusterService;
import com.bfd.mf.common.service.es.EsQueryAuthorCountService; import com.bfd.mf.common.service.es.EsQueryAuthorCountService;
import com.bfd.mf.common.service.es.EsQueryAuthorService; import com.bfd.mf.common.service.es.EsQueryAuthorService;
import com.bfd.mf.common.service.es.SubjectQueryDataService; import com.bfd.mf.common.service.es.SubjectQueryDataService;
@ -13,6 +12,7 @@ import com.bfd.mf.common.web.repository.mysql.SentimentRepository;
import com.bfd.mf.common.web.repository.mysql.base.SiteRepository; import com.bfd.mf.common.web.repository.mysql.base.SiteRepository;
import com.bfd.mf.common.web.vo.params.QueryRequest; import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity; import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity;
import com.bfd.mf.config.BFDApiConfig;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -28,7 +28,7 @@ import java.util.Map;
public class SearchAuthorService extends CrudService<SentimentModify, SentimentRepository> implements Serializable { public class SearchAuthorService extends CrudService<SentimentModify, SentimentRepository> implements Serializable {
private static Logger logger = LoggerFactory.getLogger(SearchAuthorService.class); private static Logger logger = LoggerFactory.getLogger(SearchAuthorService.class);
@Autowired @Autowired
private ClusterService clusterService;
private BFDApiConfig bfdApiConfig;
@Autowired @Autowired
private SubjectQueryDataService subjectQueryDataService; private SubjectQueryDataService subjectQueryDataService;
@Autowired @Autowired
@ -75,8 +75,8 @@ public class SearchAuthorService extends CrudService<SentimentModify, SentimentR
public JSONObject queryAuthorByAuthorId(QueryRequest queryRequest) { public JSONObject queryAuthorByAuthorId(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
try { try {
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type);
String indexName = cluster.getPrefixIndexPattern() +"_"+ queryRequest.getSubjectId();
//Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type);
String indexName = bfdApiConfig.getIndexNamePre()+ queryRequest.getSubjectId();
String[] indexNames = {indexName}; String[] indexNames = {indexName};
List<JSONObject> dataList = esQueryAuthorService.queryAuthorByAuthorId(indexNames, queryRequest); List<JSONObject> dataList = esQueryAuthorService.queryAuthorByAuthorId(indexNames, queryRequest);
jsonObject = parseAuthorMessage(dataList); jsonObject = parseAuthorMessage(dataList);
@ -135,8 +135,8 @@ public class SearchAuthorService extends CrudService<SentimentModify, SentimentR
public JSONObject queryContentsByAuthorId(QueryRequest queryRequest) { public JSONObject queryContentsByAuthorId(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
try { try {
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type);
String indexName = cluster.getPrefixIndexPattern()+ "_" + queryRequest.getSubjectId();
// Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type);
String indexName =bfdApiConfig.getIndexNamePre()+ queryRequest.getSubjectId();
String indexNames[] = {indexName}; String indexNames[] = {indexName};
List<JSONObject> dataList = esQueryAuthorService.queryContentsByAuthorId(indexNames, queryRequest); List<JSONObject> dataList = esQueryAuthorService.queryContentsByAuthorId(indexNames, queryRequest);
List<ESMonitorEntity> esMonitorEntityLists = new ArrayList<>(); List<ESMonitorEntity> esMonitorEntityLists = new ArrayList<>();
@ -280,8 +280,8 @@ public class SearchAuthorService extends CrudService<SentimentModify, SentimentR
} }
private String[] getIndexNames(QueryRequest queryRequest) { private String[] getIndexNames(QueryRequest queryRequest) {
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
// Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
String[] indexName = currentIndexList.toArray(new String[currentIndexList.size()]); String[] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
return indexName; return indexName;
} }

443
cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java
View File

@ -37,8 +37,6 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
@Autowired @Autowired
private BFDApiConfig bfdApiConfig; private BFDApiConfig bfdApiConfig;
@Autowired @Autowired
private ClusterService clusterService;
@Autowired
private EsCommonService esCommonService; private EsCommonService esCommonService;
@Autowired @Autowired
private ESServerUtils esServerUtils; private ESServerUtils esServerUtils;
@ -56,13 +54,13 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
return null; return null;
} }
private SearchResponse buildDataIdQueryCrawl(Integer from,Integer searchSize,List<String> dataIdList,
String orderFlag,String sortFlag,
private SearchResponse buildDataIdQueryCrawl(Integer from, Integer searchSize, List<String> dataIdList,
String orderFlag, String sortFlag,
List<String> currentIndexList, Cluster cluster) { List<String> currentIndexList, Cluster cluster) {
if(sortFlag.equals(ESConstant.COMMENT)){
if (sortFlag.equals(ESConstant.COMMENT)) {
sortFlag = ESConstant.COMMENTS_COUNT; sortFlag = ESConstant.COMMENTS_COUNT;
} }
if(sortFlag.equals("")){
if (sortFlag.equals("")) {
sortFlag = ESConstant.PUBTIME; sortFlag = ESConstant.PUBTIME;
} }
QueryBuilder queryBuilder = esCommonService.buildStringQueryByField(ESConstant.DATA_ID, dataIdList); QueryBuilder queryBuilder = esCommonService.buildStringQueryByField(ESConstant.DATA_ID, dataIdList);
@ -88,19 +86,20 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
ESMonitorEntity mainMonitorEntity = parseMainMessage(mainMessageHit); ESMonitorEntity mainMonitorEntity = parseMainMessage(mainMessageHit);
esMonitorListEntity.add(mainMonitorEntity); esMonitorListEntity.add(mainMonitorEntity);
} }
}catch (Exception e){
} catch (Exception e) {
logger.info("[SearchDataService] parseQueryResult ERROR !"); logger.info("[SearchDataService] parseQueryResult ERROR !");
} }
} }
/** /**
* 遍历查询结果调用解析组装方法 2 用户信息的组装用的是这个方法！！！ * 遍历查询结果调用解析组装方法 2 用户信息的组装用的是这个方法！！！
*/ */
private void parseQueryResult(List<JSONObject> dataList, List<ESMonitorEntity> esMonitorListEntity,Integer searchType) {
private void parseQueryResult(List<JSONObject> dataList, List<ESMonitorEntity> esMonitorListEntity, Integer searchType) {
try { try {
List<Map<String, Object>> site = siteRepository.findsiteByDel(0); List<Map<String, Object>> site = siteRepository.findsiteByDel(0);
Map<String,Map<String,Object>> siteMap = new HashMap<>();
Map<String, Map<String, Object>> siteMap = new HashMap<>();
for (Map<String, Object> map : site) { for (Map<String, Object> map : site) {
siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(),map);
siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(), map);
} }
if (null != dataList && dataList.size() > 0) { if (null != dataList && dataList.size() > 0) {
for (JSONObject json : dataList) { for (JSONObject json : dataList) {
@ -108,7 +107,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
esMonitorListEntity.add(mainMonitorEntity); esMonitorListEntity.add(mainMonitorEntity);
} }
} }
}catch (Exception e){
} catch (Exception e) {
logger.info("[SearchDataService] parseQueryResult ERROR !"); logger.info("[SearchDataService] parseQueryResult ERROR !");
} }
} }
@ -120,7 +119,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
String indexName = firstHit.getIndex(); String indexName = firstHit.getIndex();
Map<String, Object> sourceAsMap = firstHit.getSourceAsMap(); Map<String, Object> sourceAsMap = firstHit.getSourceAsMap();
String title = "标题为空"; String title = "标题为空";
if(sourceAsMap.containsKey(ESConstant.TITLE)) {
if (sourceAsMap.containsKey(ESConstant.TITLE)) {
title = sourceAsMap.get(ESConstant.TITLE).toString(); title = sourceAsMap.get(ESConstant.TITLE).toString();
} }
String content = sourceAsMap.get(ESConstant.CONTENT).toString(); String content = sourceAsMap.get(ESConstant.CONTENT).toString();
@ -130,12 +129,12 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
String docType = sourceAsMap.get(ESConstant.DOC_TYPE).toString(); String docType = sourceAsMap.get(ESConstant.DOC_TYPE).toString();
String channel = sourceAsMap.get(ESConstant.CHANNEL).toString(); String channel = sourceAsMap.get(ESConstant.CHANNEL).toString();
String enSource = sourceAsMap.get(ESConstant.EN_SOURCE).toString(); String enSource = sourceAsMap.get(ESConstant.EN_SOURCE).toString();
if (docType.equals(ESConstant.SOCIAL) && !enSource.equals("weixin") ) {
if (docType.equals(ESConstant.SOCIAL) && !enSource.equals("weixin")) {
title = sourceAsMap.get(ESConstant.AUTHOR).toString(); title = sourceAsMap.get(ESConstant.AUTHOR).toString();
} }
String authorId = ""; String authorId = "";
if(sourceAsMap.containsKey(ESConstant.AUTHORID)){
if (sourceAsMap.containsKey(ESConstant.AUTHORID)) {
authorId = sourceAsMap.get(ESConstant.AUTHORID).toString(); authorId = sourceAsMap.get(ESConstant.AUTHORID).toString();
} }
String url = sourceAsMap.get(ESConstant.URL).toString(); String url = sourceAsMap.get(ESConstant.URL).toString();
@ -148,17 +147,17 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
List videoPath = (List) sourceAsMap.get(ESConstant.VIDEOPATH); List videoPath = (List) sourceAsMap.get(ESConstant.VIDEOPATH);
boolean is = true; boolean is = true;
String isDownload = sourceAsMap.get(ESConstant.ISDOWNLOAD).toString(); String isDownload = sourceAsMap.get(ESConstant.ISDOWNLOAD).toString();
if(!isDownload.equals("true")){
if (!isDownload.equals("true")) {
is = false; is = false;
} }
String vodeoUrl = sourceAsMap.get(ESConstant.VIDEOURL).toString(); String vodeoUrl = sourceAsMap.get(ESConstant.VIDEOURL).toString();
Double sentiment = 0.5; Double sentiment = 0.5;
if(sourceAsMap.containsKey(ESConstant.SYS_SENTIMENT) && !sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString().contains("0.0")) {
if (sourceAsMap.containsKey(ESConstant.SYS_SENTIMENT) && !sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString().contains("0.0")) {
sentiment = Double.valueOf(sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString()); sentiment = Double.valueOf(sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString());
} }
String forumScore = ""; String forumScore = "";
if(sourceAsMap.containsKey("forumScore")){
if (sourceAsMap.containsKey("forumScore")) {
forumScore = (String) sourceAsMap.get("forumScore"); forumScore = (String) sourceAsMap.get("forumScore");
} }
@ -182,7 +181,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
esMonitorEntity.setCommentsCount(Integer.valueOf(sourceAsMap.getOrDefault(ESConstant.COMMENTS_COUNT, 0).toString())); esMonitorEntity.setCommentsCount(Integer.valueOf(sourceAsMap.getOrDefault(ESConstant.COMMENTS_COUNT, 0).toString()));
esMonitorEntity.setQuoteCount(sourceAsMap.getOrDefault(ESConstant.QUOTE_COUNT, 0).toString()); esMonitorEntity.setQuoteCount(sourceAsMap.getOrDefault(ESConstant.QUOTE_COUNT, 0).toString());
esMonitorEntity.setAttitudesCount(sourceAsMap.getOrDefault(ESConstant.ATTITUDES_COUNT,"").toString());
esMonitorEntity.setAttitudesCount(sourceAsMap.getOrDefault(ESConstant.ATTITUDES_COUNT, "").toString());
esMonitorEntity.setCrawlTime(Long.valueOf(sourceAsMap.get(ESConstant.CRAWLTIME).toString())); esMonitorEntity.setCrawlTime(Long.valueOf(sourceAsMap.get(ESConstant.CRAWLTIME).toString()));
esMonitorEntity.setIsDownload(is); esMonitorEntity.setIsDownload(is);
@ -201,8 +200,8 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
/** /**
* 导出时会用到这个字段解析解析组装返回结果 2 * 导出时会用到这个字段解析解析组装返回结果 2
*/ */
private ESMonitorEntity parseMainMessage(JSONObject jsonObject,Integer searchType,
Map<String,Map<String,Object>> siteMap) throws Exception {
private ESMonitorEntity parseMainMessage(JSONObject jsonObject, Integer searchType,
Map<String, Map<String, Object>> siteMap) throws Exception {
ESMonitorEntity esMonitorEntity = new ESMonitorEntity(); ESMonitorEntity esMonitorEntity = new ESMonitorEntity();
try { try {
Map<String, Object> sourceAsMap = jsonObject; Map<String, Object> sourceAsMap = jsonObject;
@ -215,9 +214,9 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
if (sourceAsMap.containsKey(ESConstant.TITLE)) { if (sourceAsMap.containsKey(ESConstant.TITLE)) {
title = sourceAsMap.get(ESConstant.TITLE).toString(); title = sourceAsMap.get(ESConstant.TITLE).toString();
} }
if(searchType == 2){
if (searchType == 2) {
title = ""; title = "";
}else {
} else {
if (docType.equals(ESConstant.SOCIAL) && !enSource.equals("weixin")) { if (docType.equals(ESConstant.SOCIAL) && !enSource.equals("weixin")) {
title = sourceAsMap.get(ESConstant.AUTHOR).toString(); title = sourceAsMap.get(ESConstant.AUTHOR).toString();
} }
@ -241,7 +240,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
// if(enSource.equals(ESConstant.SINA)){ // if(enSource.equals(ESConstant.SINA)){
// siteId = "183"; // 微博的，我忘了为啥这个需要特殊处理。。。 // siteId = "183"; // 微博的，我忘了为啥这个需要特殊处理。。。
// }else // }else
if(siteMap.containsKey(enSource)){
if (siteMap.containsKey(enSource)) {
Map<String, Object> siteOtherMap = siteMap.get(enSource); Map<String, Object> siteOtherMap = siteMap.get(enSource);
if (siteOtherMap.containsKey("site_id")) { if (siteOtherMap.containsKey("site_id")) {
siteId = siteMap.get(enSource).get("site_id").toString(); siteId = siteMap.get(enSource).get("site_id").toString();
@ -255,10 +254,10 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
} }
String productParameter = ""; String productParameter = "";
if(sourceAsMap.containsKey(ESConstant.PRODUCTPARAMETER)){
if (sourceAsMap.containsKey(ESConstant.PRODUCTPARAMETER)) {
productParameter = sourceAsMap.get(ESConstant.PRODUCTPARAMETER).toString(); productParameter = sourceAsMap.get(ESConstant.PRODUCTPARAMETER).toString();
} }
if(sourceAsMap.containsKey(ESConstant.AUTHORNICKNAME)){
if (sourceAsMap.containsKey(ESConstant.AUTHORNICKNAME)) {
authornickname = sourceAsMap.get(ESConstant.AUTHORNICKNAME).toString(); authornickname = sourceAsMap.get(ESConstant.AUTHORNICKNAME).toString();
} }
// 电商主贴 // 电商主贴
@ -277,21 +276,21 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
quoteCount = sourceAsMap.get(ESConstant.PRICE).toString(); quoteCount = sourceAsMap.get(ESConstant.PRICE).toString();
price = sourceAsMap.get(ESConstant.PRICE).toString(); price = sourceAsMap.get(ESConstant.PRICE).toString();
attitudeCount = sourceAsMap.get(ESConstant.POSTCOUNT).toString(); attitudeCount = sourceAsMap.get(ESConstant.POSTCOUNT).toString();
} else if(searchType == 2){
} else if (searchType == 2) {
// 用户 // 用户
author = sourceAsMap.get(ESConstant.AUTHOR).toString(); author = sourceAsMap.get(ESConstant.AUTHOR).toString();
content = productParameter; content = productParameter;
}else {
} else {
// 其他主贴 // 其他主贴
content = sourceAsMap.get(ESConstant.CONTENT).toString(); content = sourceAsMap.get(ESConstant.CONTENT).toString();
author = sourceAsMap.get(ESConstant.AUTHOR).toString(); author = sourceAsMap.get(ESConstant.AUTHOR).toString();
quoteCount = sourceAsMap.get(ESConstant.QUOTE_COUNT).toString(); quoteCount = sourceAsMap.get(ESConstant.QUOTE_COUNT).toString();
collentCount = sourceAsMap.get(ESConstant.COLLE_CTCOUNT).toString(); collentCount = sourceAsMap.get(ESConstant.COLLE_CTCOUNT).toString();
viewCnt = Integer.valueOf(sourceAsMap.get(ESConstant.VIEW_CNT).toString()); viewCnt = Integer.valueOf(sourceAsMap.get(ESConstant.VIEW_CNT).toString());
if(sourceAsMap.get(ESConstant.ATTITUDES_COUNT).toString().contains(ESConstant.TOTALCOUNT)) {
if (sourceAsMap.get(ESConstant.ATTITUDES_COUNT).toString().contains(ESConstant.TOTALCOUNT)) {
JSONObject countMap = JSONObject.parseObject(sourceAsMap.get(ESConstant.ATTITUDES_COUNT).toString()); JSONObject countMap = JSONObject.parseObject(sourceAsMap.get(ESConstant.ATTITUDES_COUNT).toString());
attitudeCount = countMap.getString(ESConstant.TOTALCOUNT); attitudeCount = countMap.getString(ESConstant.TOTALCOUNT);
}else{
} else {
attitudeCount = sourceAsMap.get(ESConstant.ATTITUDES_COUNT).toString(); attitudeCount = sourceAsMap.get(ESConstant.ATTITUDES_COUNT).toString();
} }
price = sourceAsMap.get(ESConstant.PRICE).toString(); price = sourceAsMap.get(ESConstant.PRICE).toString();
@ -342,28 +341,28 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
vodeoUrl = sourceAsMap.get(ESConstant.VIDEOURL).toString(); vodeoUrl = sourceAsMap.get(ESConstant.VIDEOURL).toString();
} }
List filePathSize = new ArrayList(); List filePathSize = new ArrayList();
if(sourceAsMap.containsKey(ESConstant.FILEPATHSIZE)) {
if (sourceAsMap.containsKey(ESConstant.FILEPATHSIZE)) {
if (!("").equals(sourceAsMap.get(ESConstant.FILEPATHSIZE)) && null != sourceAsMap.get(ESConstant.FILEPATHSIZE)) { if (!("").equals(sourceAsMap.get(ESConstant.FILEPATHSIZE)) && null != sourceAsMap.get(ESConstant.FILEPATHSIZE)) {
// filePathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.FILEPATHSIZE).toString()); // filePathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.FILEPATHSIZE).toString());
filePathSize = (List) sourceAsMap.get(ESConstant.FILEPATHSIZE); filePathSize = (List) sourceAsMap.get(ESConstant.FILEPATHSIZE);
} }
} }
List imagePathSize = new ArrayList(); List imagePathSize = new ArrayList();
if(sourceAsMap.containsKey(ESConstant.IMAGEPATHSIZE)) {
if (sourceAsMap.containsKey(ESConstant.IMAGEPATHSIZE)) {
if (null != sourceAsMap.get(ESConstant.IMAGEPATHSIZE) && !("[]").equals(sourceAsMap.get(ESConstant.IMAGEPATHSIZE))) { if (null != sourceAsMap.get(ESConstant.IMAGEPATHSIZE) && !("[]").equals(sourceAsMap.get(ESConstant.IMAGEPATHSIZE))) {
// imagePathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.IMAGEPATHSIZE).toString()); // imagePathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.IMAGEPATHSIZE).toString());
imagePathSize = (List) sourceAsMap.get(ESConstant.IMAGEPATHSIZE); imagePathSize = (List) sourceAsMap.get(ESConstant.IMAGEPATHSIZE);
} }
} }
List videoPathSize = new ArrayList(); List videoPathSize = new ArrayList();
if(sourceAsMap.containsKey(ESConstant.VIDEOPATHSIZE)) {
if (sourceAsMap.containsKey(ESConstant.VIDEOPATHSIZE)) {
if (null != sourceAsMap.get(ESConstant.VIDEOPATHSIZE) if (null != sourceAsMap.get(ESConstant.VIDEOPATHSIZE)
&& !("[]").equals(sourceAsMap.get(ESConstant.VIDEOPATHSIZE)) && !("[]").equals(sourceAsMap.get(ESConstant.VIDEOPATHSIZE))
&& !("{\"\":null}").equals(sourceAsMap.get(ESConstant.VIDEOPATHSIZE).toString())) { && !("{\"\":null}").equals(sourceAsMap.get(ESConstant.VIDEOPATHSIZE).toString())) {
if(sourceAsMap.get(ESConstant.VIDEOPATHSIZE).toString().contains(ESConstant.URL)) {
if(sourceAsMap.get(ESConstant.VIDEOPATHSIZE) instanceof String) {
if (sourceAsMap.get(ESConstant.VIDEOPATHSIZE).toString().contains(ESConstant.URL)) {
if (sourceAsMap.get(ESConstant.VIDEOPATHSIZE) instanceof String) {
videoPathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.VIDEOPATHSIZE).toString()); videoPathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.VIDEOPATHSIZE).toString());
}else{
} else {
videoPathSize = (List) sourceAsMap.get(ESConstant.VIDEOPATHSIZE); videoPathSize = (List) sourceAsMap.get(ESConstant.VIDEOPATHSIZE);
} }
//java.lang.String cannot be cast to java.util.List //java.lang.String cannot be cast to java.util.List
@ -391,17 +390,17 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
} }
// 词云 // 词云
List<String> hlKeywords = new ArrayList<>(); List<String> hlKeywords = new ArrayList<>();
if (sourceAsMap.get(ESConstant.HL_KEYWORDS) instanceof List){
if (sourceAsMap.get(ESConstant.HL_KEYWORDS) instanceof List) {
hlKeywords = (List<String>) sourceAsMap.get(ESConstant.HL_KEYWORDS); hlKeywords = (List<String>) sourceAsMap.get(ESConstant.HL_KEYWORDS);
} }
//List<String> //List<String>
// 视频分析结果 // 视频分析结果
String asrText = ""; String asrText = "";
List<String> ocrText = new ArrayList<>(); List<String> ocrText = new ArrayList<>();
if(sourceAsMap.containsKey(ESConstant.ASRTEXT)) {
if (sourceAsMap.containsKey(ESConstant.ASRTEXT)) {
asrText = sourceAsMap.get(ESConstant.ASRTEXT).toString(); asrText = sourceAsMap.get(ESConstant.ASRTEXT).toString();
} }
if(sourceAsMap.containsKey(ESConstant.OCRTEXT)) {
if (sourceAsMap.containsKey(ESConstant.OCRTEXT)) {
ocrText = (List<String>) sourceAsMap.get(ESConstant.OCRTEXT); ocrText = (List<String>) sourceAsMap.get(ESConstant.OCRTEXT);
} }
// 如果是用户数据，需要获取下面四个字段值 // 如果是用户数据，需要获取下面四个字段值
@ -409,7 +408,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
String friendsCount = ""; String friendsCount = "";
String postCount = ""; String postCount = "";
String location = ""; String location = "";
if(searchType == 2) {
if (searchType == 2) {
if (sourceAsMap.containsKey(ESConstant.FANS_COUNT)) { if (sourceAsMap.containsKey(ESConstant.FANS_COUNT)) {
fansCount = sourceAsMap.get(ESConstant.FANS_COUNT).toString(); fansCount = sourceAsMap.get(ESConstant.FANS_COUNT).toString();
} }
@ -431,71 +430,71 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
if (sourceAsMap.containsKey(ESConstant.VALUELABEL) && null != sourceAsMap.get(ESConstant.VALUELABEL)) { if (sourceAsMap.containsKey(ESConstant.VALUELABEL) && null != sourceAsMap.get(ESConstant.VALUELABEL)) {
valueLabel = sourceAsMap.get(ESConstant.VALUELABEL).toString(); valueLabel = sourceAsMap.get(ESConstant.VALUELABEL).toString();
} }
if(sourceAsMap.containsKey(ESConstant.CATEGORYLABEL)){
if (sourceAsMap.containsKey(ESConstant.CATEGORYLABEL)) {
categoryLabel = sourceAsMap.get(ESConstant.CATEGORYLABEL).toString(); categoryLabel = sourceAsMap.get(ESConstant.CATEGORYLABEL).toString();
} }
if(sourceAsMap.containsKey(ESConstant.TAG)){
if (sourceAsMap.containsKey(ESConstant.TAG)) {
tag = sourceAsMap.get(ESConstant.TAG).toString(); tag = sourceAsMap.get(ESConstant.TAG).toString();
if(primary == 0 ){
if (primary == 0) {
String pageType = sourceAsMap.get(ESConstant.PAGETYPE).toString(); String pageType = sourceAsMap.get(ESConstant.PAGETYPE).toString();
if(pageType.equals("socialFollow")){
if (pageType.equals("socialFollow")) {
String userType = sourceAsMap.get(ESConstant.USER_TYPE).toString(); String userType = sourceAsMap.get(ESConstant.USER_TYPE).toString();
if(userType.equals( "0")){
if (userType.equals("0")) {
tag = "分享用户"; tag = "分享用户";
}else if (userType.equals("1")){
} else if (userType.equals("1")) {
tag = "点赞用户"; tag = "点赞用户";
} }
}else if (pageType.equals("socialComment")){
} else if (pageType.equals("socialComment")) {
tag = "评论用户"; tag = "评论用户";
} }
} }
} }
String otherSourceJson = ""; String otherSourceJson = "";
if(sourceAsMap.containsKey(ESConstant.OTHERSOURCEJSON)){
if (sourceAsMap.containsKey(ESConstant.OTHERSOURCEJSON)) {
otherSourceJson = sourceAsMap.get(ESConstant.OTHERSOURCEJSON).toString(); otherSourceJson = sourceAsMap.get(ESConstant.OTHERSOURCEJSON).toString();
} }
String promotionInfo = ""; // brife 字段 String promotionInfo = ""; // brife 字段
if(sourceAsMap.containsKey(ESConstant.PROMOTIONINFO)){
if (sourceAsMap.containsKey(ESConstant.PROMOTIONINFO)) {
promotionInfo = sourceAsMap.get(ESConstant.PROMOTIONINFO).toString(); promotionInfo = sourceAsMap.get(ESConstant.PROMOTIONINFO).toString();
} }
int hasFile = 0; int hasFile = 0;
int hasVideo = 0; int hasVideo = 0;
int hasImage = 0; int hasImage = 0;
if(sourceAsMap.containsKey(ESConstant.HAS_FILE)){
if (sourceAsMap.containsKey(ESConstant.HAS_FILE)) {
hasFile = (int) sourceAsMap.get(ESConstant.HAS_FILE); hasFile = (int) sourceAsMap.get(ESConstant.HAS_FILE);
} }
if(sourceAsMap.containsKey(ESConstant.HAS_VIDEO)){
if (sourceAsMap.containsKey(ESConstant.HAS_VIDEO)) {
hasVideo = (int) sourceAsMap.get(ESConstant.HAS_VIDEO); hasVideo = (int) sourceAsMap.get(ESConstant.HAS_VIDEO);
} }
if(sourceAsMap.containsKey(ESConstant.HAS_IMAGE)){
if (sourceAsMap.containsKey(ESConstant.HAS_IMAGE)) {
hasImage = (int) sourceAsMap.get(ESConstant.HAS_IMAGE); hasImage = (int) sourceAsMap.get(ESConstant.HAS_IMAGE);
} }
int readCount = 0; int readCount = 0;
if(sourceAsMap.containsKey("readCount") && sourceAsMap.get("readCount") != ""){
if (sourceAsMap.containsKey("readCount") && sourceAsMap.get("readCount") != "") {
readCount = (int) sourceAsMap.get("readCount"); readCount = (int) sourceAsMap.get("readCount");
} }
Double sentiment = 0.5; Double sentiment = 0.5;
if(sourceAsMap.containsKey(ESConstant.SYS_SENTIMENT)
if (sourceAsMap.containsKey(ESConstant.SYS_SENTIMENT)
&& !sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString().contains("0.0")) { && !sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString().contains("0.0")) {
sentiment = Double.valueOf(sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString()); sentiment = Double.valueOf(sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString());
} }
String forumScore = ""; String forumScore = "";
if(sourceAsMap.containsKey("forumScore")){
if (sourceAsMap.containsKey("forumScore")) {
forumScore = (String) sourceAsMap.get("forumScore"); forumScore = (String) sourceAsMap.get("forumScore");
} }
String userType = ""; String userType = "";
if(sourceAsMap.containsKey("userType")){
if (sourceAsMap.containsKey("userType")) {
userType = (String) sourceAsMap.get("userType"); userType = (String) sourceAsMap.get("userType");
} }
String userTypeContent = ""; String userTypeContent = "";
if(sourceAsMap.containsKey("userTypeContent")){
if (sourceAsMap.containsKey("userTypeContent")) {
userTypeContent = (String) sourceAsMap.get("userTypeContent"); userTypeContent = (String) sourceAsMap.get("userTypeContent");
} }
String pageType = ""; String pageType = "";
if(sourceAsMap.containsKey("pageType")){
if (sourceAsMap.containsKey("pageType")) {
pageType = sourceAsMap.get("pageType").toString(); pageType = sourceAsMap.get("pageType").toString();
} }
try { try {
@ -580,7 +579,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return esMonitorEntity; return esMonitorEntity;
@ -591,24 +590,22 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
Cluster cluster = null; Cluster cluster = null;
List<String> currentIndexList = new ArrayList<>(); List<String> currentIndexList = new ArrayList<>();
String subjectId = queryRequest.getSubjectId(); String subjectId = queryRequest.getSubjectId();
if(null != queryRequest.getSubjectId() && !("").equals(subjectId)){ // 如果是专题，去专题的索引查就行
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId;
if (null != queryRequest.getSubjectId() && !("").equals(subjectId)) { // 如果是专题，去专题的索引查就行
subjectId =bfdApiConfig.getIndexNamePre() + subjectId;
currentIndexList.add(subjectId); currentIndexList.add(subjectId);
}else{ // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来
} else { // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来
logger.info("[SearchDataService] queryComment: 查询 全局数据"); logger.info("[SearchDataService] queryComment: 查询 全局数据");
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
String docId = queryRequest.getDocId(); String docId = queryRequest.getDocId();
String indexType = "cl_index_"+docId.split("_")[1];
String indexType = "cl_index_" + docId.split("_")[1];
currentIndexList.add(indexType); currentIndexList.add(indexType);
} }
JSONObject result = getCommentListByDocId(queryRequest, cluster,currentIndexList);
JSONObject result = getCommentListByDocId(queryRequest, cluster, currentIndexList);
return result; return result;
} }
private JSONObject getCommentListByDocId(QueryRequest queryRequest, Cluster cluster, List<String> currentIndexList) { private JSONObject getCommentListByDocId(QueryRequest queryRequest, Cluster cluster, List<String> currentIndexList) {
JSONObject json = new JSONObject(); JSONObject json = new JSONObject();
try{
try {
String docId = queryRequest.getDocId(); String docId = queryRequest.getDocId();
/**依据文档Id查询对应的文档*/ /**依据文档Id查询对应的文档*/
QueryBuilder queryBuilder = esCommonService.buildKeyWordsQueryBuilder(docId, ESConstant.DOC_ID); QueryBuilder queryBuilder = esCommonService.buildKeyWordsQueryBuilder(docId, ESConstant.DOC_ID);
@ -621,28 +618,28 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
.must(queryBuilder1) .must(queryBuilder1)
.must(queryBuilder); .must(queryBuilder);
SortOrder flag ;
if(null !=queryRequest.getOrder() && queryRequest.getOrder().equals(ESConstant.DESC)) {
SortOrder flag;
if (null != queryRequest.getOrder() && queryRequest.getOrder().equals(ESConstant.DESC)) {
flag = SortOrder.DESC; flag = SortOrder.DESC;
}else{
} else {
flag = SortOrder.ASC; flag = SortOrder.ASC;
} }
Integer page = queryRequest.getPage(); Integer page = queryRequest.getPage();
Integer limit = queryRequest.getLimit(); Integer limit = queryRequest.getLimit();
Integer start = limit * (page-1);
Integer start = limit * (page - 1);
List<JSONObject> comments = new ArrayList<>(); List<JSONObject> comments = new ArrayList<>();
Long size = 0L; Long size = 0L;
if(null != queryRequest.getDataId() && !queryRequest.getDataId().equals("")){
if (null != queryRequest.getDataId() && !queryRequest.getDataId().equals("")) {
String dataId = queryRequest.getDataId(); String dataId = queryRequest.getDataId();
// String docType = queryRequest.getDocType(); // String docType = queryRequest.getDocType();
JSONObject TopComment = getCommentByDataId(cluster,currentIndexList,dataId,docType);
JSONObject TopComment = getCommentByDataId(cluster, currentIndexList, dataId, docType);
// System.out.println(TopComment); // System.out.println(TopComment);
if(TopComment.size() > 0) {
if (TopComment.size() > 0) {
size = 1L; size = 1L;
comments.add(TopComment); comments.add(TopComment);
} }
boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.DATA_ID,dataId));
boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.DATA_ID, dataId));
} }
String siteId = queryRequest.getSiteId(); String siteId = queryRequest.getSiteId();
@ -653,7 +650,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
.setFrom(start) .setFrom(start)
.setSize(limit) .setSize(limit)
.setFetchSource(ESConstant.COMMENT_FIELD_DATA, null) .setFetchSource(ESConstant.COMMENT_FIELD_DATA, null)
.addSort(queryRequest.getSidx(),flag);
.addSort(queryRequest.getSidx(), flag);
SearchResponse searchResponse = builder.execute().actionGet(); SearchResponse searchResponse = builder.execute().actionGet();
@ -662,23 +659,23 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
Map<String, Object> result = response[i].getSourceAsMap(); Map<String, Object> result = response[i].getSourceAsMap();
jsonObject.putAll(result); jsonObject.putAll(result);
jsonObject.put(ESConstant.SITEID,siteId);
jsonObject.put(ESConstant.SITEID, siteId);
comments.add(jsonObject); comments.add(jsonObject);
} }
json.put(ESConstant.COMMENTLISTS,comments);
json.put(ESConstant.COMMENTLISTS, comments);
size = size + searchResponse.getHits().getTotalHits(); size = size + searchResponse.getHits().getTotalHits();
json.put(ESConstant.ALLDOCNUMBER,size);
}catch (Exception e){
json.put(ESConstant.ALLDOCNUMBER, size);
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return json; return json;
} }
private JSONObject getCommentByDataId(Cluster cluster, List<String> currentIndexList, String dataId,String docType) {
private JSONObject getCommentByDataId(Cluster cluster, List<String> currentIndexList, String dataId, String docType) {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
try{
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.DATA_ID,dataId));
try {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.DATA_ID, dataId));
// 由于电商的数据的primary =1 因此不加这个条件了, // 由于电商的数据的primary =1 因此不加这个条件了,
// if(docType.equals(ESConstant.ITEM)){ // if(docType.equals(ESConstant.ITEM)){
// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.PRIMARY, 1)); // boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.PRIMARY, 1));
@ -692,12 +689,12 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
SearchResponse searchResponse = builder.execute().actionGet(); SearchResponse searchResponse = builder.execute().actionGet();
SearchHit[] response = searchResponse.getHits().getHits(); SearchHit[] response = searchResponse.getHits().getHits();
if(response.length > 0){
if (response.length > 0) {
Map<String, Object> result = response[0].getSourceAsMap(); Map<String, Object> result = response[0].getSourceAsMap();
jsonObject.putAll(result); jsonObject.putAll(result);
} }
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return jsonObject; return jsonObject;
@ -705,6 +702,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
/** /**
* 根据docId 查询一条数据的详情及评论列表 * 根据docId 查询一条数据的详情及评论列表
*
* @param queryRequest * @param queryRequest
* @return * @return
*/ */
@ -714,12 +712,10 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
Cluster cluster = null; Cluster cluster = null;
List<String> currentIndexList = new ArrayList<>(); List<String> currentIndexList = new ArrayList<>();
String subjectId = queryRequest.getSubjectId(); String subjectId = queryRequest.getSubjectId();
if(!("").equals(subjectId) && null != subjectId){ // 如果是专题，去专题的索引查就行
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId;
}else{ // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来
logger.info("[SearchDataService] queryOneDataByDocId 查询 全局数据 : {}" , subjectId);
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
if (!("").equals(subjectId) && null != subjectId) { // 如果是专题，去专题的索引查就行
subjectId = bfdApiConfig.getIndexNamePre() + subjectId;
} else { // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来
logger.info("[SearchDataService] queryOneDataByDocId 查询 全局数据 : {}", subjectId);
} }
currentIndexList.add(subjectId); currentIndexList.add(subjectId);
String docId = queryRequest.getDocId(); String docId = queryRequest.getDocId();
@ -727,74 +723,72 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
JSONObject jsonObject = getOneDataByDocId(docId, cluster, currentIndexList); JSONObject jsonObject = getOneDataByDocId(docId, cluster, currentIndexList);
jsonObject = setLable(jsonObject); jsonObject = setLable(jsonObject);
// 替换几个 pathSize 中的链接的前缀 // 替换几个 pathSize 中的链接的前缀
if(jsonObject.containsKey(ESConstant.IMAGEPATHSIZE)){
List<Map<String,String>>imagePathSize = (List<Map<String, String>>) jsonObject.get(ESConstant.IMAGEPATHSIZE);
for (Map<String,String> imagePath: imagePathSize) {
if (jsonObject.containsKey(ESConstant.IMAGEPATHSIZE)) {
List<Map<String, String>> imagePathSize = (List<Map<String, String>>) jsonObject.get(ESConstant.IMAGEPATHSIZE);
for (Map<String, String> imagePath : imagePathSize) {
String url = imagePath.get(ESConstant.URL); String url = imagePath.get(ESConstant.URL);
url = url.replace(bfdApiConfig.getGoFastDomain(),"").replace("http://172.18.1.113:8892","");
imagePath.put(ESConstant.URL,url);
url = url.replace(bfdApiConfig.getGoFastDomain(), "").replace("http://172.18.1.113:8892", "");
imagePath.put(ESConstant.URL, url);
} }
} }
if(jsonObject.containsKey(ESConstant.VIDEOPATHSIZE)){
List<Map<String,String>>videoPathSize = (List<Map<String, String>>) jsonObject.get(ESConstant.VIDEOPATHSIZE);
for (Map<String,String> videoPath: videoPathSize) {
if (jsonObject.containsKey(ESConstant.VIDEOPATHSIZE)) {
List<Map<String, String>> videoPathSize = (List<Map<String, String>>) jsonObject.get(ESConstant.VIDEOPATHSIZE);
for (Map<String, String> videoPath : videoPathSize) {
String url = videoPath.get(ESConstant.URL); String url = videoPath.get(ESConstant.URL);
url = url.replace(bfdApiConfig.getGoFastDomain(),"").replace("http://172.18.1.113:8892","");
videoPath.put(ESConstant.URL,url);
url = url.replace(bfdApiConfig.getGoFastDomain(), "").replace("http://172.18.1.113:8892", "");
videoPath.put(ESConstant.URL, url);
} }
} }
if(jsonObject.containsKey(ESConstant.FILEPATHSIZE)){ //http://172.18.1.113:8080
List<Map<String,String>>filePathSize = (List<Map<String, String>>) jsonObject.get(ESConstant.FILEPATHSIZE);
for (Map<String,String> filePath: filePathSize) {
if (jsonObject.containsKey(ESConstant.FILEPATHSIZE)) { //http://172.18.1.113:8080
List<Map<String, String>> filePathSize = (List<Map<String, String>>) jsonObject.get(ESConstant.FILEPATHSIZE);
for (Map<String, String> filePath : filePathSize) {
String url = filePath.get(ESConstant.URL); String url = filePath.get(ESConstant.URL);
url = url.replace(bfdApiConfig.getGoFastDomain(),"").replace("http://172.18.1.113:8892","");
filePath.put(ESConstant.URL,url);
url = url.replace(bfdApiConfig.getGoFastDomain(), "").replace("http://172.18.1.113:8892", "");
filePath.put(ESConstant.URL, url);
} }
} }
String enSource = jsonObject.getString(ESConstant.EN_SOURCE); String enSource = jsonObject.getString(ESConstant.EN_SOURCE);
if(null != enSource) {
if (null != enSource) {
jsonObject = getSite(jsonObject, enSource); jsonObject = getSite(jsonObject, enSource);
} }
// 如果是社交媒体类的数据，需要将 author 放到 title 字段中做显示 // 如果是社交媒体类的数据，需要将 author 放到 title 字段中做显示
jsonObject = socialDataChangeAuthorAndTitle(jsonObject); jsonObject = socialDataChangeAuthorAndTitle(jsonObject);
// 根据用户ID 查询一下用户信息，如果没有的话，就把原来的用户名放进去。 // 根据用户ID 查询一下用户信息，如果没有的话，就把原来的用户名放进去。
String docType = jsonObject.getString(ESConstant.DOC_TYPE); String docType = jsonObject.getString(ESConstant.DOC_TYPE);
jsonObject = getUserMessageByAuthorId(jsonObject,docType,queryRequest,subjectId);
jsonObject = getUserMessageByAuthorId(jsonObject, docType, queryRequest, subjectId);
// 如果是电商数据，需要把电商详情信息填到 content 中，并把一些店铺信息填到 author中 // 如果是电商数据，需要把电商详情信息填到 content 中，并把一些店铺信息填到 author中
jsonObject = getItemDataShopMeggage(jsonObject,docType);
jsonObject = getItemDataShopMeggage(jsonObject, docType);
return jsonObject; return jsonObject;
} }
// 崔老师那个版本，需要添加 分类标签和 价值标签 // 崔老师那个版本，需要添加 分类标签和 价值标签
private JSONObject setLable(JSONObject jsonObject) { private JSONObject setLable(JSONObject jsonObject) {
if(!jsonObject.containsKey(ESConstant.VALUELABEL) || null == jsonObject.get(ESConstant.VALUELABEL)){
jsonObject.put(ESConstant.VALUELABEL,"");
}else{
jsonObject.put(ESConstant.VALUELABEL,jsonObject.get(ESConstant.VALUELABEL).toString());
if (!jsonObject.containsKey(ESConstant.VALUELABEL) || null == jsonObject.get(ESConstant.VALUELABEL)) {
jsonObject.put(ESConstant.VALUELABEL, "");
} else {
jsonObject.put(ESConstant.VALUELABEL, jsonObject.get(ESConstant.VALUELABEL).toString());
} }
if(!jsonObject.containsKey(ESConstant.CATEGORYLABEL)){
jsonObject.put(ESConstant.CATEGORYLABEL,"");
if (!jsonObject.containsKey(ESConstant.CATEGORYLABEL)) {
jsonObject.put(ESConstant.CATEGORYLABEL, "");
} }
return jsonObject; return jsonObject;
} }
private JSONObject getSite(JSONObject jsonObject, String enSource) { private JSONObject getSite(JSONObject jsonObject, String enSource) {
List<Map<String, Object>> site = siteRepository.findSiteByEnSource(enSource); List<Map<String, Object>> site = siteRepository.findSiteByEnSource(enSource);
Map<String,Map<String,Object>> siteMap = new HashMap<>();
Map<String, Map<String, Object>> siteMap = new HashMap<>();
for (Map<String, Object> map : site) { for (Map<String, Object> map : site) {
siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(),map);
siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(), map);
} }
String siteId = ""; String siteId = "";
String icon = ""; String icon = "";
String siteType = ""; String siteType = "";
Map<String,Object> siteOtherMap = siteMap.get(enSource);
if(enSource.equals(ESConstant.SINA)){
Map<String, Object> siteOtherMap = siteMap.get(enSource);
if (enSource.equals(ESConstant.SINA)) {
siteId = "183"; siteId = "183";
}else {
} else {
if (siteOtherMap.containsKey("site_id")) { if (siteOtherMap.containsKey("site_id")) {
siteId = siteMap.get(enSource).get("site_id").toString(); siteId = siteMap.get(enSource).get("site_id").toString();
} }
@ -805,21 +799,21 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
siteType = siteMap.get(enSource).get("site_type").toString(); siteType = siteMap.get(enSource).get("site_type").toString();
} }
} }
jsonObject.put(ESConstant.SITEID,siteId);
jsonObject.put(ESConstant.SITETYPE,siteType);
jsonObject.put(ESConstant.SITEICON,icon);
jsonObject.put(ESConstant.SITEID, siteId);
jsonObject.put(ESConstant.SITETYPE, siteType);
jsonObject.put(ESConstant.SITEICON, icon);
return jsonObject; return jsonObject;
} }
private JSONObject getItemDataShopMeggage(JSONObject jsonObject,String docType) {
if(jsonObject.containsKey(ESConstant.DOC_TYPE) && docType.equals(ESConstant.ITEM)){
jsonObject.put(ESConstant.CONTENT,jsonObject.get(ESConstant.PRODUCTPARAMETER));
private JSONObject getItemDataShopMeggage(JSONObject jsonObject, String docType) {
if (jsonObject.containsKey(ESConstant.DOC_TYPE) && docType.equals(ESConstant.ITEM)) {
jsonObject.put(ESConstant.CONTENT, jsonObject.get(ESConstant.PRODUCTPARAMETER));
JSONObject newJsonObject = new JSONObject(); JSONObject newJsonObject = new JSONObject();
newJsonObject.put(ESConstant.AUTHOR,jsonObject.get(ESConstant.AUTHORNICKNAME));
newJsonObject.put(ESConstant.SOURCE,jsonObject.get(ESConstant.SOURCE));
newJsonObject.put(ESConstant.URL,jsonObject.get(ESConstant.URL));
jsonObject.put(ESConstant.AUTHOR,newJsonObject);
newJsonObject.put(ESConstant.AUTHOR, jsonObject.get(ESConstant.AUTHORNICKNAME));
newJsonObject.put(ESConstant.SOURCE, jsonObject.get(ESConstant.SOURCE));
newJsonObject.put(ESConstant.URL, jsonObject.get(ESConstant.URL));
jsonObject.put(ESConstant.AUTHOR, newJsonObject);
} }
return jsonObject; return jsonObject;
} }
@ -827,8 +821,8 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
/** /**
* 根据 主贴中的 authorID 查一下相关的用户信息 * 根据 主贴中的 authorID 查一下相关的用户信息
*/ */
private JSONObject getUserMessageByAuthorId(JSONObject jsonObject,String docType,QueryRequest queryRequest,String subjectId) {
if(null != docType) {
private JSONObject getUserMessageByAuthorId(JSONObject jsonObject, String docType, QueryRequest queryRequest, String subjectId) {
if (null != docType) {
JSONObject newJsonObject = new JSONObject(); JSONObject newJsonObject = new JSONObject();
newJsonObject.put(ESConstant.AUTHOR, jsonObject.get(ESConstant.AUTHOR)); newJsonObject.put(ESConstant.AUTHOR, jsonObject.get(ESConstant.AUTHOR));
jsonObject.put(ESConstant.AUTHOR, newJsonObject); jsonObject.put(ESConstant.AUTHOR, newJsonObject);
@ -840,18 +834,18 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
* 社交媒体类数据，将作者替换到 标题中做显示 * 社交媒体类数据，将作者替换到 标题中做显示
*/ */
private JSONObject socialDataChangeAuthorAndTitle(JSONObject jsonObject) { private JSONObject socialDataChangeAuthorAndTitle(JSONObject jsonObject) {
if(jsonObject.containsKey(ESConstant.DOC_TYPE)){
if(jsonObject.get(ESConstant.DOC_TYPE).equals(ESConstant.SOCIAL)){
if (jsonObject.containsKey(ESConstant.DOC_TYPE)) {
if (jsonObject.get(ESConstant.DOC_TYPE).equals(ESConstant.SOCIAL)) {
String author = jsonObject.getString(ESConstant.AUTHOR); String author = jsonObject.getString(ESConstant.AUTHOR);
String enSource = jsonObject.getString(ESConstant.EN_SOURCE); String enSource = jsonObject.getString(ESConstant.EN_SOURCE);
if(!enSource.equals("weixin")) {
if (!enSource.equals("weixin")) {
jsonObject.put(ESConstant.TITLE, author); jsonObject.put(ESConstant.TITLE, author);
// 把 content 的值 放入到 译文Title 中是为了展示一下翻译，这个后面还是删掉吧。 // 把 content 的值 放入到 译文Title 中是为了展示一下翻译，这个后面还是删掉吧。
String content = jsonObject.getString(ESConstant.CONTENT); String content = jsonObject.getString(ESConstant.CONTENT);
jsonObject.put(ESConstant.TRANSLATECONTENT,content);
jsonObject.put(ESConstant.TRANSLATECONTENT, content);
} }
if(jsonObject.get(ESConstant.EN_SOURCE).equals(ESConstant.SINA)){
jsonObject.put(ESConstant.SOURCE,"微博");
if (jsonObject.get(ESConstant.EN_SOURCE).equals(ESConstant.SINA)) {
jsonObject.put(ESConstant.SOURCE, "微博");
} }
} }
} }
@ -872,27 +866,28 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
.setFetchSource(ESConstant.FIELD_DATA, null); .setFetchSource(ESConstant.FIELD_DATA, null);
SearchResponse searchDataResponse = builder.execute().actionGet(); SearchResponse searchDataResponse = builder.execute().actionGet();
System.out.println("本次查询结果条数："+searchDataResponse.getHits().totalHits);
if(searchDataResponse.getHits().getHits().length >0) {
System.out.println("本次查询结果条数：" + searchDataResponse.getHits().totalHits);
if (searchDataResponse.getHits().getHits().length > 0) {
Map<String, Object> result = searchDataResponse.getHits().getHits()[0].getSourceAsMap(); Map<String, Object> result = searchDataResponse.getHits().getHits()[0].getSourceAsMap();
if (result.size() > 0) { result.entrySet().stream()
if (result.size() > 0) {
result.entrySet().stream()
.forEach(entry -> { .forEach(entry -> {
if (entry.getKey().equals(ESConstant.FILEPATH) && entry.getValue().equals("")) { if (entry.getKey().equals(ESConstant.FILEPATH) && entry.getValue().equals("")) {
jsonObject.put(entry.getKey(), new ArrayList<>()); jsonObject.put(entry.getKey(), new ArrayList<>());
} else if (entry.getKey().equals(ESConstant.FILEPATHSIZE) } else if (entry.getKey().equals(ESConstant.FILEPATHSIZE)
|| entry.getKey().equals(ESConstant.IMAGEPATHSIZE) || entry.getKey().equals(ESConstant.IMAGEPATHSIZE)
|| entry.getKey().equals(ESConstant.VIDEOPATHSIZE)) { || entry.getKey().equals(ESConstant.VIDEOPATHSIZE)) {
if(entry.getValue().toString() .contains(ESConstant.URL)) {
if (entry.getValue().toString().contains(ESConstant.URL)) {
jsonObject.put(entry.getKey(), entry.getValue()); jsonObject.put(entry.getKey(), entry.getValue());
}else{
jsonObject.put(entry.getKey(),new ArrayList<>());
} else {
jsonObject.put(entry.getKey(), new ArrayList<>());
} }
} else if(entry.getKey().equals(ESConstant.ATTITUDES_COUNT)){
if(entry.getValue().toString().contains(ESConstant.TOTALCOUNT)){
} else if (entry.getKey().equals(ESConstant.ATTITUDES_COUNT)) {
if (entry.getValue().toString().contains(ESConstant.TOTALCOUNT)) {
JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString()); JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString());
jsonObject.put(entry.getKey(),totalCount.get(ESConstant.TOTALCOUNT));
}else{
jsonObject.put(entry.getKey(),entry.getValue());
jsonObject.put(entry.getKey(), totalCount.get(ESConstant.TOTALCOUNT));
} else {
jsonObject.put(entry.getKey(), entry.getValue());
} }
} else { } else {
jsonObject.put(entry.getKey(), entry.getValue()); jsonObject.put(entry.getKey(), entry.getValue());
@ -912,10 +907,10 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
List<String> currentIndexList = new ArrayList<>(); List<String> currentIndexList = new ArrayList<>();
// 获取ES的参数及要查询的索引列表 // 获取ES的参数及要查询的索引列表
String subjectId = queryRequest.getSubjectId(); String subjectId = queryRequest.getSubjectId();
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 special_cluster_type
currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster,subjectId);
// cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 special_cluster_type
currentIndexList = subjectQueryDataService.getIndexBySubjectIds(subjectId);
Long clusterId = cluster.getId(); Long clusterId = cluster.getId();
logger.info("[SearchDataService] queryDataList clusterId : {}; currentIndexList : {}" ,clusterId, currentIndexList.toString());
logger.info("[SearchDataService] queryDataList clusterId : {}; currentIndexList : {}", clusterId, currentIndexList.toString());
String orderFlag = queryRequest.getOrder(); // 排序方式 asc/desc String orderFlag = queryRequest.getOrder(); // 排序方式 asc/desc
String sortFlag = queryRequest.getSidx(); // 排序字段 String sortFlag = queryRequest.getSidx(); // 排序字段
@ -927,15 +922,15 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
// 开始查询 // 开始查询
List<String> dataIdList = new ArrayList<>(); List<String> dataIdList = new ArrayList<>();
List<SearchResponse> cacheList = subjectQueryDataService.fetchResponseDataFromCache( List<SearchResponse> cacheList = subjectQueryDataService.fetchResponseDataFromCache(
queryRequest,clusterId,
queryRequest, clusterId,
orderFlag, sortFlag, orderFlag, sortFlag,
currentIndexList, currentIndexList,
ESConstant.FIELD_ID_LIST); ESConstant.FIELD_ID_LIST);
if(cacheList.size() == 0){
if (cacheList.size() == 0) {
logger.info("没有查到相关数据哦！"); logger.info("没有查到相关数据哦！");
jsonObject.put(ESConstant.ALLDOCNUMBER,foldDocAllNumber);// 实际查询总量
jsonObject.put(ESConstant.MONITORLISTS,esMonitorEntityLists);
jsonObject.put(ESConstant.ALLDOCNUMBER, foldDocAllNumber);// 实际查询总量
jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists);
return jsonObject; return jsonObject;
} }
@ -949,13 +944,13 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
List<ESMonitorBaseEntity> timeSeries = new ArrayList<>(values); List<ESMonitorBaseEntity> timeSeries = new ArrayList<>(values);
Integer limit = queryRequest.getLimit(); //每页的数量 Integer limit = queryRequest.getLimit(); //每页的数量
Integer start = (queryRequest.getPage()-1)*limit; //起始页(0,20,40....)
Integer start = (queryRequest.getPage() - 1) * limit; //起始页(0,20,40....)
// 将查询结果的 dataId 写入到 dataIdList中 // 将查询结果的 dataId 写入到 dataIdList中
Map<String,String> dedupmap = new HashMap<>();
if(timeSeries.size() > 0) {
Map<String, String> dedupmap = new HashMap<>();
if (timeSeries.size() > 0) {
for (int i = start; i < timeSeries.size(); i++) { for (int i = start; i < timeSeries.size(); i++) {
if(null != timeSeries.get(i).getDataId() && !("").equals(timeSeries.get(i).getDataId())) {
if (null != timeSeries.get(i).getDataId() && !("").equals(timeSeries.get(i).getDataId())) {
dedupmap.put(timeSeries.get(i).getDataId(), timeSeries.get(i).getDataId()); dedupmap.put(timeSeries.get(i).getDataId(), timeSeries.get(i).getDataId());
} }
} }
@ -972,7 +967,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
currentIndexList, currentIndexList,
cluster); cluster);
logger.info("Response : ",response);
logger.info("Response : ", response);
parseQueryResult(response, esMonitorEntityLists); parseQueryResult(response, esMonitorEntityLists);
Long responseStart = System.currentTimeMillis(); Long responseStart = System.currentTimeMillis();
logger.info("[SearchDataService] [responseStart] used :{}", System.currentTimeMillis() - responseStart); logger.info("[SearchDataService] [responseStart] used :{}", System.currentTimeMillis() - responseStart);
@ -983,8 +978,8 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
Collections.sort(esMonitorEntityLists); Collections.sort(esMonitorEntityLists);
// 实际查询总量 // 实际查询总量
jsonObject.put(ESConstant.ALLDOCNUMBER,foldDocAllNumber);
jsonObject.put(ESConstant.MONITORLISTS,esMonitorEntityLists);
jsonObject.put(ESConstant.ALLDOCNUMBER, foldDocAllNumber);
jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists);
return jsonObject; return jsonObject;
} }
@ -994,23 +989,23 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
public JSONObject queryDataInOneIndex(QueryRequest queryRequest) { public JSONObject queryDataInOneIndex(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
try { try {
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
Long clusterId = cluster.getId();
logger.info("[SearchDataService] queryDataInOneIndex: clusterId :{} ; currentIndexList : {}", clusterId,currentIndexList.toString());
// Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
//Long clusterId = cluster.getId();
logger.info("[SearchDataService] queryDataInOneIndex: currentIndexList : {}", currentIndexList.toString());
// String indexName = currentIndexList.get(0); // String indexName = currentIndexList.get(0);
String indexNames [] = currentIndexList.toArray(new String [currentIndexList.size()]);
String indexNames[] = currentIndexList.toArray(new String[currentIndexList.size()]);
List<JSONObject> dataList = esQueryServiceForSQMini.queryDataFromOneSubject(indexNames, queryRequest); List<JSONObject> dataList = esQueryServiceForSQMini.queryDataFromOneSubject(indexNames, queryRequest);
List<ESMonitorEntity> esMonitorEntityLists = new ArrayList<>(); List<ESMonitorEntity> esMonitorEntityLists = new ArrayList<>();
Integer searchType = queryRequest.getSearchType(); Integer searchType = queryRequest.getSearchType();
parseQueryResult(dataList, esMonitorEntityLists, searchType); parseQueryResult(dataList, esMonitorEntityLists, searchType);
Long totalCount = esQueryServiceForSQMini.queryDataCountFromOneSubject(indexNames,queryRequest);
logger.info("[SearchDataService] queryDataInOneIndex: {}",totalCount);
jsonObject.put(ESConstant.ALLDOCNUMBER,totalCount);
jsonObject.put(ESConstant.MONITORLISTS,esMonitorEntityLists);
Long totalCount = esQueryServiceForSQMini.queryDataCountFromOneSubject(indexNames, queryRequest);
logger.info("[SearchDataService] queryDataInOneIndex: {}", totalCount);
jsonObject.put(ESConstant.ALLDOCNUMBER, totalCount);
jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists);
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return jsonObject; return jsonObject;
@ -1018,6 +1013,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
/** /**
* 导出专题下数据调用的方法 * 导出专题下数据调用的方法
*
* @param queryRequest * @param queryRequest
* @return * @return
*/ */
@ -1025,35 +1021,35 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
try { try {
// 获取 ES 的连接方式及要查询的索引列表 专题索引 special_cluster_type // 获取 ES 的连接方式及要查询的索引列表 专题索引 special_cluster_type
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
Long clusterId = cluster.getId();
String [] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; clusterId :{} ; currentIndexList :{}",indexName[0], clusterId , currentIndexList.toString());
//Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
// Long clusterId = cluster.getId();
String[] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; currentIndexList :{}", indexName[0], currentIndexList.toString());
// 开始查询 // 开始查询
jsonObject= esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest);
jsonObject = esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest);
List<JSONObject> dataList = (List<JSONObject>) jsonObject.get(ESConstant.MONITORLISTS); List<JSONObject> dataList = (List<JSONObject>) jsonObject.get(ESConstant.MONITORLISTS);
List<ESMonitorEntity> esMonitorEntityLists = new ArrayList<>(); List<ESMonitorEntity> esMonitorEntityLists = new ArrayList<>();
Integer searchType = queryRequest.getSearchType(); Integer searchType = queryRequest.getSearchType();
parseQueryResult(dataList, esMonitorEntityLists,searchType);
parseQueryResult(dataList, esMonitorEntityLists, searchType);
jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists); jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists);
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return jsonObject; return jsonObject;
} }
public JSONObject exportJsonDataInSubject(QueryRequest queryRequest){
public JSONObject exportJsonDataInSubject(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
try{
try {
// 获取 ES 的连接方式及要查询的索引列表 // 获取 ES 的连接方式及要查询的索引列表
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 专题索引 special_cluster_type
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
Long clusterId = cluster.getId();
String [] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; clusterId :{} ; currentIndexList :{}",indexName[0], clusterId , currentIndexList.toString());
jsonObject= esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest);
}catch (Exception e){
// Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 专题索引 special_cluster_type
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
// Long clusterId = cluster.getId();
String[] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; currentIndexList :{}", indexName[0], currentIndexList.toString());
jsonObject = esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest);
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return jsonObject; return jsonObject;
@ -1066,23 +1062,23 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
List<String> currentIndexList = new ArrayList<>(); List<String> currentIndexList = new ArrayList<>();
// 获取ES的参数及要查询的索引列表 // 获取ES的参数及要查询的索引列表
String subjectId = queryRequest.getSubjectId(); String subjectId = queryRequest.getSubjectId();
if(subjectId.equals("all")) {
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
currentIndexList = subjectQueryDataService.getIndexListByTimeRange(cluster, queryRequest.getStartTime(), queryRequest.getEndTime());
}else if(subjectId.contains(",")){
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
if (subjectId.equals("all")) {
// cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
// currentIndexList = subjectQueryDataService.getIndexListByTimeRange(queryRequest.getStartTime(), queryRequest.getEndTime());
} else if (subjectId.contains(",")) {
// cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
} }
Long clusterId = cluster.getId(); Long clusterId = cluster.getId();
logger.info("[SearchDataService] exportDataInOneIndex: clusterId :{}; currentIndexList : {}" , clusterId , currentIndexList.toString());
logger.info("[SearchDataService] exportDataInOneIndex: clusterId :{}; currentIndexList : {}", clusterId, currentIndexList.toString());
// 开始查询 // 开始查询
jsonObject= esQueryServiceForSQNormal.exportDataFromIndexs(currentIndexList, queryRequest);
jsonObject = esQueryServiceForSQNormal.exportDataFromIndexs(currentIndexList, queryRequest);
List<JSONObject> dataList = (List<JSONObject>) jsonObject.get(ESConstant.MONITORLISTS); List<JSONObject> dataList = (List<JSONObject>) jsonObject.get(ESConstant.MONITORLISTS);
List<ESMonitorEntity> esMonitorEntityLists = new ArrayList<>(); List<ESMonitorEntity> esMonitorEntityLists = new ArrayList<>();
Integer searchType = queryRequest.getSearchType(); Integer searchType = queryRequest.getSearchType();
parseQueryResult(dataList, esMonitorEntityLists,searchType);
jsonObject.put(ESConstant.MONITORLISTS,esMonitorEntityLists);
}catch (Exception e){
parseQueryResult(dataList, esMonitorEntityLists, searchType);
jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists);
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return jsonObject; return jsonObject;
@ -1090,13 +1086,10 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
public JSONObject queryDataCountsInOneIndex(QueryRequest queryRequest) { public JSONObject queryDataCountsInOneIndex(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
try{
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 special_cluster_type
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
Long clusterId = cluster.getId();
logger.info("[SearchDataService] queryDataCountsInOneIndex: clusterId : {}; currentIndexList : {}" , clusterId ,currentIndexList.toString());
// String indexName = currentIndexList.get(0);
String indexNames [] = currentIndexList.toArray(new String [currentIndexList.size()]);
try {
String subjectId = queryRequest.getSubjectId();
String indexName = bfdApiConfig.getIndexNamePre()+subjectId;
String indexNames[] = {indexName};
Long contentCount = 0L; Long contentCount = 0L;
Long commentCount = 0L; Long commentCount = 0L;
Long authorCount = 0L; Long authorCount = 0L;
@ -1107,13 +1100,13 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
commentCount = esQueryServiceForSQMini.queryDataCountFromOneSubject(indexNames, queryRequest); commentCount = esQueryServiceForSQMini.queryDataCountFromOneSubject(indexNames, queryRequest);
queryRequest.setSearchType(2); queryRequest.setSearchType(2);
authorCount = esQueryServiceForSQMini.queryDataCountFromOneSubject(indexNames, queryRequest); authorCount = esQueryServiceForSQMini.queryDataCountFromOneSubject(indexNames, queryRequest);
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
jsonObject.put(ESConstant.CONTENTCOUNT,contentCount);
jsonObject.put(ESConstant.COMMENTCOUNT,commentCount);
jsonObject.put(ESConstant.AUTHORCOUNT,authorCount);
}catch (Exception e){
jsonObject.put(ESConstant.CONTENTCOUNT, contentCount);
jsonObject.put(ESConstant.COMMENTCOUNT, commentCount);
jsonObject.put(ESConstant.AUTHORCOUNT, authorCount);
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return jsonObject; return jsonObject;
@ -1128,9 +1121,9 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
String newIndex = newIndexPre + queryRequest.getSubjectId(); String newIndex = newIndexPre + queryRequest.getSubjectId();
try { try {
// 将上面专题的数据复制到新的专题下！ // 将上面专题的数据复制到新的专题下！
long created = esQueryServiceForSQMini.reIndexData(oldIndex,newIndex);
jsonObject.put("created",created);
}catch (Exception e){
long created = esQueryServiceForSQMini.reIndexData(oldIndex, newIndex);
jsonObject.put("created", created);
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return jsonObject; return jsonObject;
@ -1138,10 +1131,10 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
public JSONObject deleteBySubjectId(QueryRequest queryRequest) { public JSONObject deleteBySubjectId(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
String indexName = bfdApiConfig.getIndexNamePre()+queryRequest.getSubjectId();
String indexName = bfdApiConfig.getIndexNamePre() + queryRequest.getSubjectId();
try { try {
esQueryServiceForSQMini.deleteBySubjectId(indexName); esQueryServiceForSQMini.deleteBySubjectId(indexName);
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return jsonObject; return jsonObject;
@ -1149,11 +1142,11 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
public JSONObject deleteBySubjectIdByCid(QueryRequest queryRequest) { public JSONObject deleteBySubjectIdByCid(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
String indexName = bfdApiConfig.getIndexNamePre()+ queryRequest.getSubjectId();
String indexName = bfdApiConfig.getIndexNamePre() + queryRequest.getSubjectId();
String cid = queryRequest.getCid(); String cid = queryRequest.getCid();
try { try {
esQueryServiceForSQMini.deleteBySubjectIdByCid(indexName,cid);
}catch (Exception e){
esQueryServiceForSQMini.deleteBySubjectIdByCid(indexName, cid);
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return jsonObject; return jsonObject;
@ -1161,12 +1154,12 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
public JSONObject deleteBySubjectIdByCrawlDataFlag(QueryRequest queryRequest) { public JSONObject deleteBySubjectIdByCrawlDataFlag(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
String indexName = bfdApiConfig.getIndexNamePre()+ queryRequest.getSubjectId();
String indexName = bfdApiConfig.getIndexNamePre() + queryRequest.getSubjectId();
String cid = queryRequest.getCid(); String cid = queryRequest.getCid();
String crawlDataFlag = queryRequest.getCrawlDataFlag(); String crawlDataFlag = queryRequest.getCrawlDataFlag();
try { try {
esQueryServiceForSQMini.deleteBySubjectIdByCrawlDataFlag(indexName,cid,crawlDataFlag);
}catch (Exception e){
esQueryServiceForSQMini.deleteBySubjectIdByCrawlDataFlag(indexName, cid, crawlDataFlag);
} catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return jsonObject; return jsonObject;

6
cl_search_api/src/main/java/com/bfd/mf/service/UpdateService.java
View File

@ -2,7 +2,6 @@ package com.bfd.mf.service;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.service.common.CrudService; import com.bfd.mf.common.service.common.CrudService;
import com.bfd.mf.common.service.es.ClusterService;
import com.bfd.mf.common.util.ESServerUtils; import com.bfd.mf.common.util.ESServerUtils;
import com.bfd.mf.common.util.es.EsUtils; import com.bfd.mf.common.util.es.EsUtils;
import com.bfd.mf.common.web.entity.mysql.SentimentModify; import com.bfd.mf.common.web.entity.mysql.SentimentModify;
@ -29,8 +28,6 @@ public class UpdateService extends CrudService<SentimentModify, SentimentReposit
@Autowired @Autowired
private BFDApiConfig bfdApiConfig; private BFDApiConfig bfdApiConfig;
@Autowired @Autowired
private ClusterService clusterService;
@Autowired
private ESServerUtils esServerUtils; private ESServerUtils esServerUtils;
private String clusterName =""; private String clusterName ="";
@ -60,8 +57,7 @@ public class UpdateService extends CrudService<SentimentModify, SentimentReposit
List<String> currentIndexList = new ArrayList<>(); List<String> currentIndexList = new ArrayList<>();
// 如果是专题，去专题的索引查就行 // 如果是专题，去专题的索引查就行
if(!("").equals(subjectId) && null != subjectId){ if(!("").equals(subjectId) && null != subjectId){
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type);
subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId;
subjectId = bfdApiConfig.getIndexNamePre()+ subjectId;
} }
currentIndexList.add(subjectId); currentIndexList.add(subjectId);
Map<String, Object> params = new HashMap<>(16); Map<String, Object> params = new HashMap<>(16);

37
cl_search_api/src/main/resources/application.yml
View File

@ -1,5 +1,5 @@
server: server:
port: 18909
port: 18902
tomcat: tomcat:
uri-encoding: UTF-8 uri-encoding: UTF-8
max-threads: 800 max-threads: 800
@ -15,12 +15,16 @@ server:
spring: spring:
datasource: datasource:
driver-class-name: com.mysql.cj.jdbc.Driver driver-class-name: com.mysql.cj.jdbc.Driver
username: crawl
password: crawl123
url: jdbc:mysql://172.18.1.134:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC
# username: crawl
# password: crawl123
# url: jdbc:mysql://172.18.1.134:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC
# username: root # username: root
# password: bfd123 # password: bfd123
# url: jdbc:mysql://172.26.11.113:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC # url: jdbc:mysql://172.26.11.113:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC
username: crawl
password: crawl123
url: jdbc:mysql://172.26.11.110:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC
hikari: hikari:
maximum-pool-size: 10 maximum-pool-size: 10
minimum-idle: 1 minimum-idle: 1
@ -46,16 +50,31 @@ bfd.api.mf:
uploadZipPath : /opt/nfsdata/uploadFiles/ uploadZipPath : /opt/nfsdata/uploadFiles/
indexNamePre : cl_major_ indexNamePre : cl_major_
es-mini:
name: SQ_Mini_2
address: 172.18.1.81:9301
upper: 2018-09-01
standby: cl_major_*
# es-mini:
# name: CL_Mini_2
# address: 172.18.1.81:9301
# upper: 2018-09-01
# standby: cl_major_*
es-normal: es-normal:
name: SQ_Normal_new name: SQ_Normal_new
address: 172.18.1.134:9301 address: 172.18.1.134:9301
upper: 2018-09-01 upper: 2018-09-01
standby: cl_index_* standby: cl_index_*
# es-mini: #这个是自己部署的6.8.23 的ES
# name: my-application
# address: 172.16.10.39:9300
# upper: 2018-09-01
# standby: cl_major_*
# es-mini: # 这个是庆鹏那边搞的 6.8.13 且需要用户验证的
# name: elasticsearch
# address: 172.16.10.42:9300
# upper: 2018-09-01
# standby: cl_major_*
es-mini: # 这个是28赛博项目阿里云上的ES
name: crawl
address: 47.254.148.208:9300
upper: 2018-09-01
standby: cl_major_*
# es-mini: # es-mini:

Write Preview
Loading…
Cancel
Save