diff --git a/cl_search_api/pom.xml b/cl_search_api/pom.xml
index f0ddb5c..9decd64 100644
--- a/cl_search_api/pom.xml
+++ b/cl_search_api/pom.xml
@@ -13,7 +13,7 @@
cl_search_api
Search V3.2 API
cl_search_api
- 3.2.4-SNAPSHOT
+ 3.2.5-SNAPSHOT
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java
index 20a92b6..12b8798 100644
--- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java
+++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java
@@ -55,7 +55,8 @@ public class ParseSearchScopeService {
public static BoolQueryBuilder getSearchTypeQuery(Integer searchType) {
BoolQueryBuilder searchScopeQuery = null;
if(searchType == 0 ){ //0:主贴;1:评论;2:用户 || ES 中 primary=1为主贴
- searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY, 1));
+ searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY,
+ 1));
} else if(searchType == 1){
searchScopeQuery = QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery(ESConstant.PRIMARY, 0))
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java
index 3f38359..d56757b 100644
--- a/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java
+++ b/cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java
@@ -7,6 +7,7 @@ import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.util.thread.SubjectDataQueryThread;
import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
import com.bfd.mf.common.web.vo.params.QueryRequest;
+import com.bfd.mf.config.BFDApiConfig;
import com.bfd.nlp.common.util.string.TStringUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
@@ -38,6 +39,9 @@ public class SubjectQueryDataService {
private Logger logger = LoggerFactory.getLogger(SubjectQueryDataService.class);
private static final Long ONE_DAY = 60 * 60 * 1000L * 24;
+
+ @Autowired
+ private BFDApiConfig bfdApiConfig;
@Autowired
private TopicQueryService topicQueryService;
@Autowired
@@ -156,4 +160,22 @@ public class SubjectQueryDataService {
return new ArrayList<>(set);
}
+ /**
+ * 现在用这个方法
+ * @param subjectIds
+ * @return
+ */
+ public List getIndexBySubjectIds(String subjectIds) {
+ Set set = new HashSet<>();
+ if(subjectIds.contains(",")){
+ for(String subjectId :subjectIds.split(",")){
+ set.add(bfdApiConfig.getIndexNamePre() + subjectId);
+ }
+ }else {
+ String subjectId = subjectIds;
+ set.add(bfdApiConfig.getIndexNamePre() + subjectId);
+ }
+ return new ArrayList<>(set);
+ }
+
}
\ No newline at end of file
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java
index 0123d7c..a6d633a 100644
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java
+++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java
@@ -42,7 +42,7 @@ import java.util.Map;
*/
@Component
public class ESServerUtils implements Serializable {
- private static Logger logger = LoggerFactory.getLogger(ESServerUtils.class);
+ private final static Logger logger = LoggerFactory.getLogger(ESServerUtils.class);
private static final long serialVersionUID = 1L;
private static Map clientByClusterId = new HashMap<>();
@Autowired
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java
index 53a46d7..fe5ebb3 100644
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java
+++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java
@@ -241,11 +241,16 @@ public abstract class EsUtils {
.setQuery(queryBuilder)
.addAggregation(aggregation);
- // System.out.println(requestBuilder);
+ System.out.println(requestBuilder);
Aggregations aggregations = requestBuilder.get().getAggregations();
Cardinality cardinality = aggregations.get(count);
- // System.out.println("1111 : "+cardinality.getValue());
- return cardinality.getValue();
+ System.out.println("1111 : " + cardinality.getValue());
+ System.out.println("2222 : " + requestBuilder.get().getHits().totalHits);
+ long resultCount = cardinality.getValue();
+ if(searchType == 2){
+ resultCount = requestBuilder.get().getHits().totalHits;
+ }
+ return resultCount;
}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java
index 87af8d9..1acbf49 100644
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java
+++ b/cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java
@@ -1,6 +1,5 @@
package com.bfd.mf.common.util.slice;
-import com.bfd.mf.common.service.es.ClusterService;
import com.bfd.mf.common.service.es.SubjectQueryDataService;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
@@ -25,8 +24,6 @@ import java.util.stream.Collectors;
public class SliceScrollUtil {
private static final Logger logger = LoggerFactory.getLogger(SliceScrollUtil.class);
@Autowired
- private ClusterService clusterService;
- @Autowired
private SubjectQueryDataService subjectQueryDataService;
/**
@@ -131,12 +128,12 @@ public class SliceScrollUtil {
List currentIndexList = null;
if(null != queryRequest.getSubjectId() && !("all").equals(queryRequest.getSubjectId())){
logger.info("查询 【专题数据】 subjectId :{}" ,queryRequest.getSubjectId());
- cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
- currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
- }else{
- logger.info("[SliceScrollUtil] fetchResultSubjectCache : 查询 【全局数据】");
- cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
- currentIndexList = subjectQueryDataService.getIndexListByTimeRange(cluster, queryRequest.getStartTime(),queryRequest.getEndTime());
+ //cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
+ currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
+// }else{
+// logger.info("[SliceScrollUtil] fetchResultSubjectCache : 查询 【全局数据】");
+// cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
+// currentIndexList = subjectQueryDataService.getIndexListByTimeRange(queryRequest.getStartTime(),queryRequest.getEndTime());
}
Long clusterId = cluster.getId();
diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java
index 107d181..ccb2366 100644
--- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java
+++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java
@@ -2,7 +2,6 @@ package com.bfd.mf.service;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.service.common.CrudService;
-import com.bfd.mf.common.service.es.ClusterService;
import com.bfd.mf.common.service.es.EsQueryAuthorCountService;
import com.bfd.mf.common.service.es.EsQueryAuthorService;
import com.bfd.mf.common.service.es.SubjectQueryDataService;
@@ -13,6 +12,7 @@ import com.bfd.mf.common.web.repository.mysql.SentimentRepository;
import com.bfd.mf.common.web.repository.mysql.base.SiteRepository;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity;
+import com.bfd.mf.config.BFDApiConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -28,7 +28,7 @@ import java.util.Map;
public class SearchAuthorService extends CrudService implements Serializable {
private static Logger logger = LoggerFactory.getLogger(SearchAuthorService.class);
@Autowired
- private ClusterService clusterService;
+ private BFDApiConfig bfdApiConfig;
@Autowired
private SubjectQueryDataService subjectQueryDataService;
@Autowired
@@ -75,8 +75,8 @@ public class SearchAuthorService extends CrudService dataList = esQueryAuthorService.queryAuthorByAuthorId(indexNames, queryRequest);
jsonObject = parseAuthorMessage(dataList);
@@ -135,8 +135,8 @@ public class SearchAuthorService extends CrudService dataList = esQueryAuthorService.queryContentsByAuthorId(indexNames, queryRequest);
List esMonitorEntityLists = new ArrayList<>();
@@ -280,8 +280,8 @@ public class SearchAuthorService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
+ // Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
+ List currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
String[] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
return indexName;
}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java b/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java
index 3e726b6..0ae01a7 100644
--- a/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java
+++ b/cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java
@@ -37,8 +37,6 @@ public class SearchDataService extends CrudService dataIdList,
- String orderFlag,String sortFlag,
+ private SearchResponse buildDataIdQueryCrawl(Integer from, Integer searchSize, List dataIdList,
+ String orderFlag, String sortFlag,
List currentIndexList, Cluster cluster) {
- if(sortFlag.equals(ESConstant.COMMENT)){
+ if (sortFlag.equals(ESConstant.COMMENT)) {
sortFlag = ESConstant.COMMENTS_COUNT;
}
- if(sortFlag.equals("")){
+ if (sortFlag.equals("")) {
sortFlag = ESConstant.PUBTIME;
}
QueryBuilder queryBuilder = esCommonService.buildStringQueryByField(ESConstant.DATA_ID, dataIdList);
@@ -88,19 +86,20 @@ public class SearchDataService extends CrudService dataList, List esMonitorListEntity,Integer searchType) {
+ private void parseQueryResult(List dataList, List esMonitorListEntity, Integer searchType) {
try {
List> site = siteRepository.findsiteByDel(0);
- Map> siteMap = new HashMap<>();
+ Map> siteMap = new HashMap<>();
for (Map map : site) {
- siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(),map);
+ siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(), map);
}
if (null != dataList && dataList.size() > 0) {
for (JSONObject json : dataList) {
@@ -108,7 +107,7 @@ public class SearchDataService extends CrudService sourceAsMap = firstHit.getSourceAsMap();
String title = "标题为空";
- if(sourceAsMap.containsKey(ESConstant.TITLE)) {
+ if (sourceAsMap.containsKey(ESConstant.TITLE)) {
title = sourceAsMap.get(ESConstant.TITLE).toString();
}
String content = sourceAsMap.get(ESConstant.CONTENT).toString();
@@ -130,12 +129,12 @@ public class SearchDataService extends CrudService> siteMap) throws Exception {
+ private ESMonitorEntity parseMainMessage(JSONObject jsonObject, Integer searchType,
+ Map> siteMap) throws Exception {
ESMonitorEntity esMonitorEntity = new ESMonitorEntity();
try {
Map sourceAsMap = jsonObject;
@@ -215,9 +214,9 @@ public class SearchDataService extends CrudService siteOtherMap = siteMap.get(enSource);
if (siteOtherMap.containsKey("site_id")) {
siteId = siteMap.get(enSource).get("site_id").toString();
@@ -255,10 +254,10 @@ public class SearchDataService extends CrudService hlKeywords = new ArrayList<>();
- if (sourceAsMap.get(ESConstant.HL_KEYWORDS) instanceof List){
+ if (sourceAsMap.get(ESConstant.HL_KEYWORDS) instanceof List) {
hlKeywords = (List) sourceAsMap.get(ESConstant.HL_KEYWORDS);
}
//List
// 视频分析结果
String asrText = "";
List ocrText = new ArrayList<>();
- if(sourceAsMap.containsKey(ESConstant.ASRTEXT)) {
+ if (sourceAsMap.containsKey(ESConstant.ASRTEXT)) {
asrText = sourceAsMap.get(ESConstant.ASRTEXT).toString();
}
- if(sourceAsMap.containsKey(ESConstant.OCRTEXT)) {
+ if (sourceAsMap.containsKey(ESConstant.OCRTEXT)) {
ocrText = (List) sourceAsMap.get(ESConstant.OCRTEXT);
}
// 如果是用户数据，需要获取下面四个字段值
@@ -409,7 +408,7 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>();
String subjectId = queryRequest.getSubjectId();
- if(null != queryRequest.getSubjectId() && !("").equals(subjectId)){ // 如果是专题，去专题的索引查就行
- cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
- subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId;
+ if (null != queryRequest.getSubjectId() && !("").equals(subjectId)) { // 如果是专题，去专题的索引查就行
+ subjectId =bfdApiConfig.getIndexNamePre() + subjectId;
currentIndexList.add(subjectId);
- }else{ // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来
+ } else { // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来
logger.info("[SearchDataService] queryComment: 查询 全局数据");
- cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
String docId = queryRequest.getDocId();
- String indexType = "cl_index_"+docId.split("_")[1];
+ String indexType = "cl_index_" + docId.split("_")[1];
currentIndexList.add(indexType);
}
- JSONObject result = getCommentListByDocId(queryRequest, cluster,currentIndexList);
+ JSONObject result = getCommentListByDocId(queryRequest, cluster, currentIndexList);
return result;
}
private JSONObject getCommentListByDocId(QueryRequest queryRequest, Cluster cluster, List currentIndexList) {
JSONObject json = new JSONObject();
- try{
+ try {
String docId = queryRequest.getDocId();
/**依据文档Id查询对应的文档*/
QueryBuilder queryBuilder = esCommonService.buildKeyWordsQueryBuilder(docId, ESConstant.DOC_ID);
@@ -621,28 +618,28 @@ public class SearchDataService extends CrudService comments = new ArrayList<>();
Long size = 0L;
- if(null != queryRequest.getDataId() && !queryRequest.getDataId().equals("")){
+ if (null != queryRequest.getDataId() && !queryRequest.getDataId().equals("")) {
String dataId = queryRequest.getDataId();
// String docType = queryRequest.getDocType();
- JSONObject TopComment = getCommentByDataId(cluster,currentIndexList,dataId,docType);
+ JSONObject TopComment = getCommentByDataId(cluster, currentIndexList, dataId, docType);
// System.out.println(TopComment);
- if(TopComment.size() > 0) {
+ if (TopComment.size() > 0) {
size = 1L;
comments.add(TopComment);
}
- boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.DATA_ID,dataId));
+ boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.DATA_ID, dataId));
}
String siteId = queryRequest.getSiteId();
@@ -653,7 +650,7 @@ public class SearchDataService extends CrudService result = response[i].getSourceAsMap();
jsonObject.putAll(result);
- jsonObject.put(ESConstant.SITEID,siteId);
+ jsonObject.put(ESConstant.SITEID, siteId);
comments.add(jsonObject);
}
- json.put(ESConstant.COMMENTLISTS,comments);
+ json.put(ESConstant.COMMENTLISTS, comments);
size = size + searchResponse.getHits().getTotalHits();
- json.put(ESConstant.ALLDOCNUMBER,size);
- }catch (Exception e){
+ json.put(ESConstant.ALLDOCNUMBER, size);
+ } catch (Exception e) {
e.printStackTrace();
}
return json;
}
- private JSONObject getCommentByDataId(Cluster cluster, List currentIndexList, String dataId,String docType) {
+ private JSONObject getCommentByDataId(Cluster cluster, List currentIndexList, String dataId, String docType) {
JSONObject jsonObject = new JSONObject();
- try{
- BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.DATA_ID,dataId));
+ try {
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.DATA_ID, dataId));
// 由于电商的数据的primary =1 因此不加这个条件了,
// if(docType.equals(ESConstant.ITEM)){
// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.PRIMARY, 1));
@@ -692,19 +689,20 @@ public class SearchDataService extends CrudService 0){
+ if (response.length > 0) {
Map result = response[0].getSourceAsMap();
jsonObject.putAll(result);
}
- }catch (Exception e){
+ } catch (Exception e) {
e.printStackTrace();
}
return jsonObject;
}
/**
- * 根据docId 查询一条数据的详情及评论列表
+ * 根据docId 查询一条数据的详情及评论列表
+ *
* @param queryRequest
* @return
*/
@@ -714,87 +712,83 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>();
String subjectId = queryRequest.getSubjectId();
- if(!("").equals(subjectId) && null != subjectId){ // 如果是专题，去专题的索引查就行
- cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
- subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId;
- }else{ // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来
- logger.info("[SearchDataService] queryOneDataByDocId 查询 全局数据 : {}" , subjectId);
- cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
+ if (!("").equals(subjectId) && null != subjectId) { // 如果是专题，去专题的索引查就行
+ subjectId = bfdApiConfig.getIndexNamePre() + subjectId;
+ } else { // 如果是全部数据，就直接去 渠道对应的索引查，渠道可以从 docId 中截取出来
+ logger.info("[SearchDataService] queryOneDataByDocId 查询 全局数据 : {}", subjectId);
}
currentIndexList.add(subjectId);
String docId = queryRequest.getDocId();
// 根据ID 获取一条详情数据
- JSONObject jsonObject = getOneDataByDocId(docId, cluster, currentIndexList);
+ JSONObject jsonObject = getOneDataByDocId(docId, cluster, currentIndexList);
jsonObject = setLable(jsonObject);
// 替换几个 pathSize 中的链接的前缀
- if(jsonObject.containsKey(ESConstant.IMAGEPATHSIZE)){
- List>imagePathSize = (List>) jsonObject.get(ESConstant.IMAGEPATHSIZE);
- for (Map imagePath: imagePathSize) {
+ if (jsonObject.containsKey(ESConstant.IMAGEPATHSIZE)) {
+ List> imagePathSize = (List>) jsonObject.get(ESConstant.IMAGEPATHSIZE);
+ for (Map imagePath : imagePathSize) {
String url = imagePath.get(ESConstant.URL);
- url = url.replace(bfdApiConfig.getGoFastDomain(),"").replace("http://172.18.1.113:8892","");
- imagePath.put(ESConstant.URL,url);
+ url = url.replace(bfdApiConfig.getGoFastDomain(), "").replace("http://172.18.1.113:8892", "");
+ imagePath.put(ESConstant.URL, url);
}
}
- if(jsonObject.containsKey(ESConstant.VIDEOPATHSIZE)){
- List>videoPathSize = (List>) jsonObject.get(ESConstant.VIDEOPATHSIZE);
- for (Map videoPath: videoPathSize) {
+ if (jsonObject.containsKey(ESConstant.VIDEOPATHSIZE)) {
+ List> videoPathSize = (List>) jsonObject.get(ESConstant.VIDEOPATHSIZE);
+ for (Map videoPath : videoPathSize) {
String url = videoPath.get(ESConstant.URL);
- url = url.replace(bfdApiConfig.getGoFastDomain(),"").replace("http://172.18.1.113:8892","");
- videoPath.put(ESConstant.URL,url);
+ url = url.replace(bfdApiConfig.getGoFastDomain(), "").replace("http://172.18.1.113:8892", "");
+ videoPath.put(ESConstant.URL, url);
}
}
- if(jsonObject.containsKey(ESConstant.FILEPATHSIZE)){ //http://172.18.1.113:8080
- List>filePathSize = (List>) jsonObject.get(ESConstant.FILEPATHSIZE);
- for (Map filePath: filePathSize) {
+ if (jsonObject.containsKey(ESConstant.FILEPATHSIZE)) { //http://172.18.1.113:8080
+ List> filePathSize = (List>) jsonObject.get(ESConstant.FILEPATHSIZE);
+ for (Map filePath : filePathSize) {
String url = filePath.get(ESConstant.URL);
- url = url.replace(bfdApiConfig.getGoFastDomain(),"").replace("http://172.18.1.113:8892","");
- filePath.put(ESConstant.URL,url);
+ url = url.replace(bfdApiConfig.getGoFastDomain(), "").replace("http://172.18.1.113:8892", "");
+ filePath.put(ESConstant.URL, url);
}
}
String enSource = jsonObject.getString(ESConstant.EN_SOURCE);
- if(null != enSource) {
+ if (null != enSource) {
jsonObject = getSite(jsonObject, enSource);
}
// 如果是社交媒体类的数据，需要将 author 放到 title 字段中做显示
jsonObject = socialDataChangeAuthorAndTitle(jsonObject);
// 根据用户ID 查询一下用户信息，如果没有的话，就把原来的用户名放进去。
String docType = jsonObject.getString(ESConstant.DOC_TYPE);
- jsonObject = getUserMessageByAuthorId(jsonObject,docType,queryRequest,subjectId);
+ jsonObject = getUserMessageByAuthorId(jsonObject, docType, queryRequest, subjectId);
// 如果是电商数据，需要把电商详情信息填到 content 中，并把一些店铺信息填到 author中
- jsonObject = getItemDataShopMeggage(jsonObject,docType);
+ jsonObject = getItemDataShopMeggage(jsonObject, docType);
return jsonObject;
}
-
-
// 崔老师那个版本，需要添加 分类标签和 价值标签
private JSONObject setLable(JSONObject jsonObject) {
- if(!jsonObject.containsKey(ESConstant.VALUELABEL) || null == jsonObject.get(ESConstant.VALUELABEL)){
- jsonObject.put(ESConstant.VALUELABEL,"");
- }else{
- jsonObject.put(ESConstant.VALUELABEL,jsonObject.get(ESConstant.VALUELABEL).toString());
+ if (!jsonObject.containsKey(ESConstant.VALUELABEL) || null == jsonObject.get(ESConstant.VALUELABEL)) {
+ jsonObject.put(ESConstant.VALUELABEL, "");
+ } else {
+ jsonObject.put(ESConstant.VALUELABEL, jsonObject.get(ESConstant.VALUELABEL).toString());
}
- if(!jsonObject.containsKey(ESConstant.CATEGORYLABEL)){
- jsonObject.put(ESConstant.CATEGORYLABEL,"");
+ if (!jsonObject.containsKey(ESConstant.CATEGORYLABEL)) {
+ jsonObject.put(ESConstant.CATEGORYLABEL, "");
}
return jsonObject;
}
private JSONObject getSite(JSONObject jsonObject, String enSource) {
List> site = siteRepository.findSiteByEnSource(enSource);
- Map> siteMap = new HashMap<>();
+ Map> siteMap = new HashMap<>();
for (Map map : site) {
- siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(),map);
+ siteMap.put(map.get(ESConstant.CID).toString().toLowerCase(), map);
}
String siteId = "";
String icon = "";
String siteType = "";
- Map siteOtherMap = siteMap.get(enSource);
- if(enSource.equals(ESConstant.SINA)){
+ Map siteOtherMap = siteMap.get(enSource);
+ if (enSource.equals(ESConstant.SINA)) {
siteId = "183";
- }else {
+ } else {
if (siteOtherMap.containsKey("site_id")) {
siteId = siteMap.get(enSource).get("site_id").toString();
}
@@ -805,21 +799,21 @@ public class SearchDataService extends CrudService0) {
+ System.out.println("本次查询结果条数：" + searchDataResponse.getHits().totalHits);
+ if (searchDataResponse.getHits().getHits().length > 0) {
Map result = searchDataResponse.getHits().getHits()[0].getSourceAsMap();
- if (result.size() > 0) { result.entrySet().stream()
- .forEach(entry -> {
- if (entry.getKey().equals(ESConstant.FILEPATH) && entry.getValue().equals("")) {
- jsonObject.put(entry.getKey(), new ArrayList<>());
- } else if (entry.getKey().equals(ESConstant.FILEPATHSIZE)
- || entry.getKey().equals(ESConstant.IMAGEPATHSIZE)
- || entry.getKey().equals(ESConstant.VIDEOPATHSIZE)) {
- if(entry.getValue().toString() .contains(ESConstant.URL)) {
+ if (result.size() > 0) {
+ result.entrySet().stream()
+ .forEach(entry -> {
+ if (entry.getKey().equals(ESConstant.FILEPATH) && entry.getValue().equals("")) {
+ jsonObject.put(entry.getKey(), new ArrayList<>());
+ } else if (entry.getKey().equals(ESConstant.FILEPATHSIZE)
+ || entry.getKey().equals(ESConstant.IMAGEPATHSIZE)
+ || entry.getKey().equals(ESConstant.VIDEOPATHSIZE)) {
+ if (entry.getValue().toString().contains(ESConstant.URL)) {
+ jsonObject.put(entry.getKey(), entry.getValue());
+ } else {
+ jsonObject.put(entry.getKey(), new ArrayList<>());
+ }
+ } else if (entry.getKey().equals(ESConstant.ATTITUDES_COUNT)) {
+ if (entry.getValue().toString().contains(ESConstant.TOTALCOUNT)) {
+ JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString());
+ jsonObject.put(entry.getKey(), totalCount.get(ESConstant.TOTALCOUNT));
+ } else {
+ jsonObject.put(entry.getKey(), entry.getValue());
+ }
+ } else {
jsonObject.put(entry.getKey(), entry.getValue());
- }else{
- jsonObject.put(entry.getKey(),new ArrayList<>());
}
- } else if(entry.getKey().equals(ESConstant.ATTITUDES_COUNT)){
- if(entry.getValue().toString().contains(ESConstant.TOTALCOUNT)){
- JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString());
- jsonObject.put(entry.getKey(),totalCount.get(ESConstant.TOTALCOUNT));
- }else{
- jsonObject.put(entry.getKey(),entry.getValue());
- }
- } else {
- jsonObject.put(entry.getKey(), entry.getValue());
- }
- });
+ });
}
}
return jsonObject;
@@ -912,10 +907,10 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>();
// 获取ES的参数及要查询的索引列表
String subjectId = queryRequest.getSubjectId();
- cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 special_cluster_type
- currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster,subjectId);
+ // cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 special_cluster_type
+ currentIndexList = subjectQueryDataService.getIndexBySubjectIds(subjectId);
Long clusterId = cluster.getId();
- logger.info("[SearchDataService] queryDataList clusterId : {}; currentIndexList : {}" ,clusterId, currentIndexList.toString());
+ logger.info("[SearchDataService] queryDataList clusterId : {}; currentIndexList : {}", clusterId, currentIndexList.toString());
String orderFlag = queryRequest.getOrder(); // 排序方式 asc/desc
String sortFlag = queryRequest.getSidx(); // 排序字段
@@ -927,15 +922,15 @@ public class SearchDataService extends CrudService dataIdList = new ArrayList<>();
List cacheList = subjectQueryDataService.fetchResponseDataFromCache(
- queryRequest,clusterId,
+ queryRequest, clusterId,
orderFlag, sortFlag,
currentIndexList,
ESConstant.FIELD_ID_LIST);
- if(cacheList.size() == 0){
+ if (cacheList.size() == 0) {
logger.info("没有查到相关数据哦！");
- jsonObject.put(ESConstant.ALLDOCNUMBER,foldDocAllNumber);// 实际查询总量
- jsonObject.put(ESConstant.MONITORLISTS,esMonitorEntityLists);
+ jsonObject.put(ESConstant.ALLDOCNUMBER, foldDocAllNumber);// 实际查询总量
+ jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists);
return jsonObject;
}
@@ -949,13 +944,13 @@ public class SearchDataService extends CrudService timeSeries = new ArrayList<>(values);
Integer limit = queryRequest.getLimit(); //每页的数量
- Integer start = (queryRequest.getPage()-1)*limit; //起始页(0,20,40....)
+ Integer start = (queryRequest.getPage() - 1) * limit; //起始页(0,20,40....)
// 将查询结果的 dataId 写入到 dataIdList中
- Map dedupmap = new HashMap<>();
- if(timeSeries.size() > 0) {
+ Map dedupmap = new HashMap<>();
+ if (timeSeries.size() > 0) {
for (int i = start; i < timeSeries.size(); i++) {
- if(null != timeSeries.get(i).getDataId() && !("").equals(timeSeries.get(i).getDataId())) {
+ if (null != timeSeries.get(i).getDataId() && !("").equals(timeSeries.get(i).getDataId())) {
dedupmap.put(timeSeries.get(i).getDataId(), timeSeries.get(i).getDataId());
}
}
@@ -972,7 +967,7 @@ public class SearchDataService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
- Long clusterId = cluster.getId();
- logger.info("[SearchDataService] queryDataInOneIndex: clusterId :{} ; currentIndexList : {}", clusterId,currentIndexList.toString());
+ // Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
+ List currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
+ //Long clusterId = cluster.getId();
+ logger.info("[SearchDataService] queryDataInOneIndex: currentIndexList : {}", currentIndexList.toString());
// String indexName = currentIndexList.get(0);
- String indexNames [] = currentIndexList.toArray(new String [currentIndexList.size()]);
+ String indexNames[] = currentIndexList.toArray(new String[currentIndexList.size()]);
List dataList = esQueryServiceForSQMini.queryDataFromOneSubject(indexNames, queryRequest);
List esMonitorEntityLists = new ArrayList<>();
Integer searchType = queryRequest.getSearchType();
parseQueryResult(dataList, esMonitorEntityLists, searchType);
- Long totalCount = esQueryServiceForSQMini.queryDataCountFromOneSubject(indexNames,queryRequest);
- logger.info("[SearchDataService] queryDataInOneIndex: {}",totalCount);
- jsonObject.put(ESConstant.ALLDOCNUMBER,totalCount);
- jsonObject.put(ESConstant.MONITORLISTS,esMonitorEntityLists);
+ Long totalCount = esQueryServiceForSQMini.queryDataCountFromOneSubject(indexNames, queryRequest);
+ logger.info("[SearchDataService] queryDataInOneIndex: {}", totalCount);
+ jsonObject.put(ESConstant.ALLDOCNUMBER, totalCount);
+ jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists);
- }catch (Exception e){
+ } catch (Exception e) {
e.printStackTrace();
}
return jsonObject;
}
/**
- * 导出专题下数据调用的方法
+ * 导出专题下数据调用的方法
+ *
* @param queryRequest
* @return
*/
@@ -1025,38 +1021,38 @@ public class SearchDataService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
- Long clusterId = cluster.getId();
- String [] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
- logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; clusterId :{} ; currentIndexList :{}",indexName[0], clusterId , currentIndexList.toString());
+ //Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
+ List currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
+ // Long clusterId = cluster.getId();
+ String[] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
+ logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; currentIndexList :{}", indexName[0], currentIndexList.toString());
// 开始查询
- jsonObject= esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest);
+ jsonObject = esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest);
List dataList = (List) jsonObject.get(ESConstant.MONITORLISTS);
List esMonitorEntityLists = new ArrayList<>();
Integer searchType = queryRequest.getSearchType();
- parseQueryResult(dataList, esMonitorEntityLists,searchType);
+ parseQueryResult(dataList, esMonitorEntityLists, searchType);
jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists);
- }catch (Exception e){
+ } catch (Exception e) {
e.printStackTrace();
}
return jsonObject;
}
- public JSONObject exportJsonDataInSubject(QueryRequest queryRequest){
+ public JSONObject exportJsonDataInSubject(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject();
- try{
+ try {
// 获取 ES 的连接方式及要查询的索引列表
- Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 专题索引 special_cluster_type
- List currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
- Long clusterId = cluster.getId();
- String [] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
- logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; clusterId :{} ; currentIndexList :{}",indexName[0], clusterId , currentIndexList.toString());
- jsonObject= esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest);
- }catch (Exception e){
+ // Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 专题索引 special_cluster_type
+ List currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
+ // Long clusterId = cluster.getId();
+ String[] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
+ logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; currentIndexList :{}", indexName[0], currentIndexList.toString());
+ jsonObject = esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest);
+ } catch (Exception e) {
e.printStackTrace();
}
- return jsonObject;
+ return jsonObject;
}
public JSONObject exportDataInDateIndex(QueryRequest queryRequest) {
@@ -1066,23 +1062,23 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>();
// 获取ES的参数及要查询的索引列表
String subjectId = queryRequest.getSubjectId();
- if(subjectId.equals("all")) {
- cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
- currentIndexList = subjectQueryDataService.getIndexListByTimeRange(cluster, queryRequest.getStartTime(), queryRequest.getEndTime());
- }else if(subjectId.contains(",")){
- cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
- currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
+ if (subjectId.equals("all")) {
+ // cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
+ // currentIndexList = subjectQueryDataService.getIndexListByTimeRange(queryRequest.getStartTime(), queryRequest.getEndTime());
+ } else if (subjectId.contains(",")) {
+ // cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
+ currentIndexList = subjectQueryDataService.getIndexBySubjectIds( queryRequest.getSubjectId());
}
Long clusterId = cluster.getId();
- logger.info("[SearchDataService] exportDataInOneIndex: clusterId :{}; currentIndexList : {}" , clusterId , currentIndexList.toString());
+ logger.info("[SearchDataService] exportDataInOneIndex: clusterId :{}; currentIndexList : {}", clusterId, currentIndexList.toString());
// 开始查询
- jsonObject= esQueryServiceForSQNormal.exportDataFromIndexs(currentIndexList, queryRequest);
+ jsonObject = esQueryServiceForSQNormal.exportDataFromIndexs(currentIndexList, queryRequest);
List dataList = (List) jsonObject.get(ESConstant.MONITORLISTS);
List esMonitorEntityLists = new ArrayList<>();
Integer searchType = queryRequest.getSearchType();
- parseQueryResult(dataList, esMonitorEntityLists,searchType);
- jsonObject.put(ESConstant.MONITORLISTS,esMonitorEntityLists);
- }catch (Exception e){
+ parseQueryResult(dataList, esMonitorEntityLists, searchType);
+ jsonObject.put(ESConstant.MONITORLISTS, esMonitorEntityLists);
+ } catch (Exception e) {
e.printStackTrace();
}
return jsonObject;
@@ -1090,13 +1086,10 @@ public class SearchDataService extends CrudService currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
- Long clusterId = cluster.getId();
- logger.info("[SearchDataService] queryDataCountsInOneIndex: clusterId : {}; currentIndexList : {}" , clusterId ,currentIndexList.toString());
- // String indexName = currentIndexList.get(0);
- String indexNames [] = currentIndexList.toArray(new String [currentIndexList.size()]);
+ try {
+ String subjectId = queryRequest.getSubjectId();
+ String indexName = bfdApiConfig.getIndexNamePre()+subjectId;
+ String indexNames[] = {indexName};
Long contentCount = 0L;
Long commentCount = 0L;
Long authorCount = 0L;
@@ -1107,13 +1100,13 @@ public class SearchDataService extends CrudService currentIndexList = new ArrayList<>();
// 如果是专题，去专题的索引查就行
if(!("").equals(subjectId) && null != subjectId){
- cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type);
- subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId;
+ subjectId = bfdApiConfig.getIndexNamePre()+ subjectId;
}
currentIndexList.add(subjectId);
Map params = new HashMap<>(16);
diff --git a/cl_search_api/src/main/resources/application.yml b/cl_search_api/src/main/resources/application.yml
index 58a1028..82dacbc 100644
--- a/cl_search_api/src/main/resources/application.yml
+++ b/cl_search_api/src/main/resources/application.yml
@@ -1,5 +1,5 @@
server:
- port: 18909
+ port: 18902
tomcat:
uri-encoding: UTF-8
max-threads: 800
@@ -15,12 +15,16 @@ server:
spring:
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
- username: crawl
- password: crawl123
- url: jdbc:mysql://172.18.1.134:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC
+# username: crawl
+# password: crawl123
+# url: jdbc:mysql://172.18.1.134:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC
# username: root
# password: bfd123
# url: jdbc:mysql://172.26.11.113:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC
+ username: crawl
+ password: crawl123
+ url: jdbc:mysql://172.26.11.110:3306/intelligent_crawl?useSSL=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC
+
hikari:
maximum-pool-size: 10
minimum-idle: 1
@@ -46,16 +50,31 @@ bfd.api.mf:
uploadZipPath : /opt/nfsdata/uploadFiles/
indexNamePre : cl_major_
- es-mini:
- name: SQ_Mini_2
- address: 172.18.1.81:9301
- upper: 2018-09-01
- standby: cl_major_*
+# es-mini:
+# name: CL_Mini_2
+# address: 172.18.1.81:9301
+# upper: 2018-09-01
+# standby: cl_major_*
es-normal:
name: SQ_Normal_new
address: 172.18.1.134:9301
upper: 2018-09-01
standby: cl_index_*
+# es-mini: #这个是自己部署的6.8.23 的ES
+# name: my-application
+# address: 172.16.10.39:9300
+# upper: 2018-09-01
+# standby: cl_major_*
+# es-mini: # 这个是庆鹏那边搞的 6.8.13 且需要用户验证的
+# name: elasticsearch
+# address: 172.16.10.42:9300
+# upper: 2018-09-01
+# standby: cl_major_*
+ es-mini: # 这个是28赛博项目阿里云上的ES
+ name: crawl
+ address: 47.254.148.208:9300
+ upper: 2018-09-01
+ standby: cl_major_*
# es-mini: