Browse Source

20220421版本-精简了一下

release-1.0
jing.du 3 years ago
parent
commit
3d82868826
  1. 2
      .idea/compiler.xml
  2. 2
      .idea/modules.xml
  3. 5
      cl_query_data_job/pom.xml
  4. 4
      cl_search_api/cl_search_api.iml
  5. 9
      cl_search_api/pom.xml
  6. 250
      cl_search_api/src/main/java/com/bfd/mf/common/service/cache/TopicQueryService.java
  7. 4
      cl_search_api/src/main/java/com/bfd/mf/common/service/common/CrudService.java
  8. 4
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/ClusterService.java
  9. 7
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsCommonService.java
  10. 2
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorCountService.java
  11. 23
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorService.java
  12. 75
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQMini.java
  13. 2
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQNormal.java
  14. 12
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/GetQueryBuilder.java
  15. 2
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java
  16. 5
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java
  17. 2
      cl_search_api/src/main/java/com/bfd/mf/common/service/text/TextService.java
  18. 4
      cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java
  19. 1
      cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java
  20. 13
      cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java
  21. 261
      cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java
  22. 2
      cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils2.java
  23. 3
      cl_search_api/src/main/java/com/bfd/mf/common/util/es/MonitorUtils.java
  24. 10
      cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java
  25. 2
      cl_search_api/src/main/java/com/bfd/mf/common/util/utility/DateUtil.java
  26. 46
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/cache/Cluster.java
  27. 1
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/topic/ParseExcelTask.java
  28. 469
      cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/UserRepository.java
  29. 1
      cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/topic/ParseExcelTaskRepository.java
  30. 2
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/QueryRequest.java
  31. 88
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataLineCount.java
  32. 145
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorBaseEntity.java
  33. 2
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorEntity.java
  34. 9
      cl_search_api/src/main/java/com/bfd/mf/controller/SearchAnalysisController.java
  35. 3
      cl_search_api/src/main/java/com/bfd/mf/controller/SearchAuthorController.java
  36. 127
      cl_search_api/src/main/java/com/bfd/mf/controller/SearchDataController.java
  37. 8
      cl_search_api/src/main/java/com/bfd/mf/controller/UploadExcelController.java
  38. 9
      cl_search_api/src/main/java/com/bfd/mf/service/SearchAnalysisService.java
  39. 5
      cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java
  40. 271
      cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java
  41. 18
      cl_search_api/src/main/java/com/bfd/mf/service/SearchKeywordsCouldService.java
  42. 8
      cl_search_api/src/main/java/com/bfd/mf/service/UpdateService.java
  43. 11
      cl_search_api/src/main/java/com/bfd/mf/service/UploadExcelService.java
  44. 12
      cl_stream_3.1.2.iml
  45. 4
      pom.xml

2
.idea/compiler.xml

@ -14,8 +14,8 @@
<bytecodeTargetLevel>
<module name="cl_query_data_job" target="1.8" />
<module name="cl_search_api" target="1.8" />
<module name="cl_stream_3" target="1.8" />
<module name="cl_stream_3.1.1" target="1.8" />
<module name="cl_stream_3.1.2" target="1.8" />
</bytecodeTargetLevel>
</component>
</project>

2
.idea/modules.xml

@ -4,7 +4,7 @@
<modules>
<module fileurl="file://$PROJECT_DIR$/cl_query_data_job/cl_query_data_job.iml" filepath="$PROJECT_DIR$/cl_query_data_job/cl_query_data_job.iml" />
<module fileurl="file://$PROJECT_DIR$/cl_search_api/cl_search_api.iml" filepath="$PROJECT_DIR$/cl_search_api/cl_search_api.iml" />
<module fileurl="file://$PROJECT_DIR$/cl_stream_3.1.2.iml" filepath="$PROJECT_DIR$/cl_stream_3.1.2.iml" />
<module fileurl="file://$PROJECT_DIR$/cl_stream_3.iml" filepath="$PROJECT_DIR$/cl_stream_3.iml" />
</modules>
</component>
</project>

5
cl_query_data_job/pom.xml

@ -4,9 +4,9 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>cl_stream_3.1.2</artifactId>
<artifactId>cl_stream_3.2</artifactId>
<groupId>com.bfd.mf</groupId>
<version>3.1.2-SNAPSHOT</version>
<version>3.2-SNAPSHOT</version>
</parent>
<artifactId>cl_query_data_job</artifactId>
@ -232,6 +232,7 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.0.0.RELEASE</version>
<configuration>
<mainClass>com.bfd.mf.job.Application</mainClass>
</configuration>

4
cl_search_api/cl_search_api.iml

@ -14,7 +14,9 @@
<configuration>
<setting name="validation-enabled" value="true" />
<setting name="provider-name" value="Hibernate" />
<datasource-mapping />
<datasource-mapping>
<factory-entry name="entityManagerFactory" />
</datasource-mapping>
<naming-strategy-map />
</configuration>
</facet>

9
cl_search_api/pom.xml

@ -5,15 +5,15 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>cl_stream_3.1.2</artifactId>
<artifactId>cl_stream_3.2</artifactId>
<groupId>com.bfd.mf</groupId>
<version>3.1.2-SNAPSHOT</version>
<version>3.2-SNAPSHOT</version>
</parent>
<name>cl_search_api</name>
<description>Search V3.1.2 API</description>
<description>Search V3.2 API</description>
<artifactId>cl_search_api</artifactId>
<version>3.1.2-SNAPSHOT</version>
<version>3.2-SNAPSHOT</version>
@ -278,6 +278,7 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>2.0.0.RELEASE</version>
<configuration>
<mainClass>com.bfd.mf.SearchApplication</mainClass>
</configuration>

250
cl_search_api/src/main/java/com/bfd/mf/common/service/cache/TopicQueryService.java

@ -9,17 +9,23 @@ import com.bfd.mf.common.web.repository.mysql.base.SiteRepository;
import com.bfd.mf.common.web.repository.mysql.topic.TaskRepository;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.nlp.common.util.object.TObjectUtils;
import org.apache.log4j.Logger;
import org.elasticsearch.index.query.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Service
public class TopicQueryService {
private static Logger logger = Logger.getLogger(TopicQueryService.class);
private static Logger logger = LoggerFactory.getLogger(TopicQueryService.class);
@Autowired
private EsCommonService esCommonService;
@Autowired
@ -32,28 +38,28 @@ public class TopicQueryService {
* 条件查询语句组装 渠道站点地区时间任务标识等
*/
public BoolQueryBuilder queryByConditions_v1(QueryRequest queryRequest) {
logger.info("[TopicQueryService] queryByConditions_v1 " );
logger.info("[TopicQueryService] queryByConditions_v1 ");
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
try {
// 价值标签
if(!("").equals(queryRequest.getValueLabel()) && null != queryRequest.getValueLabel()){
QueryBuilder valueLableQuery = QueryBuilders.termQuery(ESConstant.VALUELABEL+".keyword",queryRequest.getValueLabel());
if (!("").equals(queryRequest.getValueLabel()) && null != queryRequest.getValueLabel()) {
QueryBuilder valueLableQuery = QueryBuilders.termQuery(ESConstant.VALUELABEL + ".keyword", queryRequest.getValueLabel());
boolQuery.must(valueLableQuery);
}
// 分类标签
// 因为是多选因此需要判断如果是多个的话需要先用, 分隔然后再用模糊匹配进行查询
if(!("").equals(queryRequest.getCategoryLabel()) && null != queryRequest.getCategoryLabel()){
// QueryBuilder categoryLableQuery = QueryBuilders.queryStringQuery(queryRequest.getCategoryLabel()).defaultField(ESConstant.CATEGORYLABEL); //+".keyword"
if(queryRequest.getCategoryLabel().contains(",")){ // 有多个标签查询用should
String labels []= queryRequest.getCategoryLabel().split(",");
if (!("").equals(queryRequest.getCategoryLabel()) && null != queryRequest.getCategoryLabel()) {
// QueryBuilder categoryLableQuery = QueryBuilders.queryStringQuery(queryRequest.getCategoryLabel()).defaultField(ESConstant.CATEGORYLABEL); //+".keyword"
if (queryRequest.getCategoryLabel().contains(",")) { // 有多个标签查询用should
String labels[] = queryRequest.getCategoryLabel().split(",");
BoolQueryBuilder shouleLaelQuery = QueryBuilders.boolQuery();
for (String label:labels) {
QueryBuilder categoryLableQuery = QueryBuilders.matchPhraseQuery(ESConstant.CATEGORYLABEL,label);
for (String label : labels) {
QueryBuilder categoryLableQuery = QueryBuilders.matchPhraseQuery(ESConstant.CATEGORYLABEL, label);
shouleLaelQuery.should(categoryLableQuery);
boolQuery.must(shouleLaelQuery);
}
}else{
QueryBuilder categoryLableQuery = QueryBuilders.matchPhraseQuery(ESConstant.CATEGORYLABEL,queryRequest.getCategoryLabel());
} else {
QueryBuilder categoryLableQuery = QueryBuilders.matchPhraseQuery(ESConstant.CATEGORYLABEL, queryRequest.getCategoryLabel());
boolQuery.must(categoryLableQuery);
}
}
@ -64,7 +70,7 @@ public class TopicQueryService {
} else {
List<String> areaList = siteRepository.findCidsByArea(queryRequest.getSearchArea());
List lowCaseAreaList = areaList.stream().map(String::toLowerCase).collect(Collectors.toList());
boolQuery.must(QueryBuilders.termsQuery(ESConstant.EN_SOURCE+".keyword",lowCaseAreaList));
boolQuery.must(QueryBuilders.termsQuery(ESConstant.EN_SOURCE, lowCaseAreaList));
// String searchArea = getSearchArea(queryRequest.getSearchArea());
// boolQuery.must(QueryBuilders.termQuery(ESConstant.AREA, searchArea));
}
@ -79,9 +85,9 @@ public class TopicQueryService {
// cid 站点
String cid = queryRequest.getCid();
// taskId 任务ID
if(null == queryRequest.getTaskId() || ("").equals(queryRequest.getTaskId()) ){
if (null == queryRequest.getTaskId() || ("").equals(queryRequest.getTaskId())) {
logger.info("[TopicQueryService] queryByConditions_v1 没有任务ID,查询专题下全部任务");
}else{
} else {
String taskId = queryRequest.getTaskId();
// 如果有任务ID 的话查这个任务ID 对应的 crawlDataFlag 及时间范围然后写到查询条件中
Task task = taskRepository.findAllBydel0AndTaskId(taskId);
@ -95,11 +101,11 @@ public class TopicQueryService {
if (null == cid || ("").equals(cid) || ("test").equals(cid)) {
logger.info("[TopicQueryService] queryByConditions_v1 查询全部站点");
} else {
if(cid.equals("Tmall") || cid.equals("Taobao")){
boolQuery.must(QueryBuilders.termsQuery(ESConstant.EN_SOURCE+".keyword", "tmall","taobao"));
// boolQuery.should(QueryBuilders.termQuery(ESConstant.EN_SOURCE+".keyword", "taobao"));
}else {
boolQuery.must(QueryBuilders.termQuery(ESConstant.EN_SOURCE+".keyword", cid.toLowerCase()));
if (cid.equals("Tmall") || cid.equals("Taobao")) {
boolQuery.must(QueryBuilders.termsQuery(ESConstant.EN_SOURCE, "tmall", "taobao"));
// boolQuery.should(QueryBuilders.termQuery(ESConstant.EN_SOURCE, "taobao"));
} else {
boolQuery.must(QueryBuilders.termQuery(ESConstant.EN_SOURCE, cid.toLowerCase()));
}
}
@ -112,7 +118,7 @@ public class TopicQueryService {
}
Integer searchType = queryRequest.getSearchType();
if(searchType == 2){
if (searchType == 2) {
return boolQuery;
}
// pubtime && crawlTime 发表时间抓取时间
@ -121,21 +127,23 @@ public class TopicQueryService {
long crawlEndTime = queryRequest.getCrawlEndTime();
boolQuery.must(QueryBuilders.rangeQuery(ESConstant.CRAWLTIME).gte(crawlStartTime).lt(crawlEndTime));
} else {
long startTime =0L;
if(null != queryRequest.getStartTime() ) {
long startTime = 0L;
if (null != queryRequest.getStartTime()) {
startTime = queryRequest.getStartTime();
}
long endTime = 0L;
if(null != queryRequest.getEndTime()) {
if (null != queryRequest.getEndTime()) {
endTime = queryRequest.getEndTime();
}
if(startTime == 0 && endTime == 0){
if (startTime == 0 && endTime == 0) {
logger.info("没有时间筛选,查询全部专题下的数据!");
}else if(startTime == 0){ // 如果开始时间为空那就查询小于 endTime 的所有数据
// 如果开始时间为空那就查询小于 endTime 的所有数据
} else if (startTime == 0) {
boolQuery.must(QueryBuilders.rangeQuery(ESConstant.PUBTIME).lt(endTime));
}else if (endTime == 0){ // 如果结束时间为空加就查大于 startTime 的所有数据
// 如果结束时间为空加就查大于 startTime 的所有数据
} else if (endTime == 0) {
boolQuery.must(QueryBuilders.rangeQuery(ESConstant.PUBTIME).gte(startTime));
}else{
} else {
boolQuery.must(QueryBuilders.rangeQuery(ESConstant.PUBTIME).gte(startTime).lt(endTime));
}
}
@ -148,49 +156,132 @@ public class TopicQueryService {
Integer imageDownload = queryRequest.getWithImage();
Integer videoDownload = queryRequest.getWithVideo();
if (null != withAppendix && withAppendix == 0) { // 选中无附件的因此 is_download = false
// 4个全=0全选 没有查询条件 查全部
// BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// if (withAppendix == 0 && fileDownload == 0 && imageDownload == 0 && videoDownload == 0) {
// // 不组装语句
// } else if (withAppendix == 1) {
// // 查有附件的,有文件 有图片 有视频
// if (fileDownload == 0 && imageDownload == 0 && videoDownload == 0) {
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
// // 有图片或者有视频
// }else if (fileDownload == 1 && imageDownload == 0 && videoDownload == 0){
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
// // 有文件或者有视频
// }else if (fileDownload == 0 && imageDownload == 1 && videoDownload == 0){
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
// // 有图片或者有文件
// }else if (fileDownload == 0 && imageDownload == 0 && videoDownload == 1){
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
// // 有视频
// }else if (fileDownload == 1 && imageDownload == 1 && videoDownload == 0){
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
// // 有图片
// }else if (fileDownload == 1 && imageDownload == 0 && videoDownload == 1){
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
// // 有文件
// }else if (fileDownload == 0 && imageDownload == 1 && videoDownload == 1){
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
// }
// } else if (withAppendix == 0) {
// // 只选择 无附件
// if (fileDownload == 1 && imageDownload == 1 && videoDownload == 1) {
// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.HAS_FILE, 0));
// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 0));
// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 0));
// // 无附件 & 有视频
// }else if(fileDownload == 1 && imageDownload == 1 && videoDownload == 0){
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 0));
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 0));
// // 无附件 & 有图片
// }else if(fileDownload == 1 && imageDownload == 0 && videoDownload == 1){
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 0));
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 0));
// //无附件 & 有文件
// }else if(fileDownload == 0 && imageDownload == 1 && videoDownload == 1){
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
// //无附件 & 有文件 无图片无视频 后面三个不对啊
// }else if(fileDownload == 1 && imageDownload == 0 && videoDownload == 0){
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
// //无附件 & 有文件
// }else if(fileDownload == 0 && imageDownload == 1 && videoDownload == 0){
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 0));
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 0));
// //无附件 & 有文件
// }else if(fileDownload == 0 && imageDownload == 0 && videoDownload == 1){
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 0));
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
// boolQueryBuilder.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 0));
// }
// }
// 无附件=0其他三个为1 fileDownload=1imageDownload=1videoDownload=1
// 无附件=0有图片 imageDownload=0fileDownload=1videoDownload=1
// 无附件=0有视频 videoDownload=0fileDownload=1imageDownload=1
// 无附件=0有文件 fileDownload=0imageDownload=1videoDownload=1
// 无附件=1 有图片有视频有附件 fileDownload=0imageDownload=0videoDownload=0
// 无附件=1 fileDownload=0imageDownload=0videoDownload=0
// 选中无附件的因此 is_download = false ==1 表示未选择
// withAppendix == 1 表示选中了 "有附件,有图片,有视频" 没有选"无附件"所以 isDownload = true
if (null != withAppendix && withAppendix == 1) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.IS_DOWNLOAD, false));
// boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1))
// .should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1))
// .should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.IS_DOWNLOAD, true));
if (null != fileDownload && fileDownload == 0) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
}
if (null != imageDownload && imageDownload == 0) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
}
if (null != videoDownload && videoDownload == 0) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
}
boolQuery.must(boolQueryBuilder);
} else { // 当选中有附件的
boolQuery.must(QueryBuilders.termQuery(ESConstant.IS_DOWNLOAD, true));
} else { // 当选中无附件的 withAppendix == 0
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
if (null != fileDownload && fileDownload == 1 && null != imageDownload && imageDownload == 1 && null != videoDownload && videoDownload == 1) {
boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1));
boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1));
boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.IS_DOWNLOAD, false));
if (null != fileDownload && fileDownload == 1
&& null != imageDownload && imageDownload == 1
&& null != videoDownload && videoDownload == 1) {
boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
boolQuery.mustNot(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
} else if (null != fileDownload && fileDownload == 0 && null != imageDownload && imageDownload == 1 && null != videoDownload && videoDownload == 1) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
} else if (null != fileDownload && fileDownload == 1 && null != imageDownload && imageDownload == 0 && null != videoDownload && videoDownload == 1) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
} else if (null != fileDownload && fileDownload == 1 && null != imageDownload && imageDownload == 1 && null != videoDownload && videoDownload == 0) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
} else if (null != fileDownload && fileDownload == 0 && null != imageDownload && imageDownload == 0 && null != videoDownload && videoDownload == 1) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
} else if (null != imageDownload && imageDownload == 0 && null != videoDownload && videoDownload == 0 && null != fileDownload && fileDownload == 1) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
} else if (null != videoDownload && videoDownload == 0 && null != fileDownload && fileDownload == 0 && null != imageDownload && imageDownload == 1) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1));
} else if (null != videoDownload && videoDownload == 0 && null != fileDownload && fileDownload == 0 && null != imageDownload && imageDownload == 0) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_EGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_UGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.SEARCH_SCOPE_PGC, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
} else if (null != videoDownload && videoDownload == 0
&& null != fileDownload && fileDownload == 0
&& null != imageDownload && imageDownload == 0) {
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_VIDEO, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_FILE, 1));
boolQueryBuilder.should(QueryBuilders.termQuery(ESConstant.HAS_IMAGE, 1));
}
boolQuery.must(boolQueryBuilder);
}
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
}
return boolQuery;
@ -206,10 +297,10 @@ public class TopicQueryService {
// primary = 1
boolQuery.must(QueryBuilders.termQuery(ESConstant.PRIMARY, 1));
// authorId =
if(null != queryRequest.getAuthorId() && !("").equals(queryRequest.getAuthorId())){
if (null != queryRequest.getAuthorId() && !("").equals(queryRequest.getAuthorId())) {
boolQuery.must(QueryBuilders.termQuery(ESConstant.AUTHORID, queryRequest.getAuthorId()));
}
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
}
return boolQuery;
@ -225,22 +316,21 @@ public class TopicQueryService {
// primary = 2
boolQuery.must(QueryBuilders.termQuery(ESConstant.PRIMARY, 2));
// authorId =
if(null != queryRequest.getAuthorId() && !("").equals(queryRequest.getAuthorId())){
if (null != queryRequest.getAuthorId() && !("").equals(queryRequest.getAuthorId())) {
boolQuery.must(QueryBuilders.termQuery(ESConstant.AUTHORID, queryRequest.getAuthorId()));
}
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
}
return boolQuery;
}
/**
* 根据 searchScope 中的查询条件组装查询语句
*/
public BoolQueryBuilder buildSearchTextBuilder(String searchScope,String keyword) {
return ParseSearchScopeService.getSearchScopeQuery(searchScope,keyword);
public BoolQueryBuilder buildSearchTextBuilder(String searchScope, String keyword) {
return ParseSearchScopeService.getSearchScopeQuery(searchScope, keyword);
}
/**
@ -268,8 +358,8 @@ public class TopicQueryService {
* 组装渠道查询语句
*/
private QueryBuilder buildChannelQueryBuilder(QueryRequest queryRequest) {
String [] channel = queryRequest.getSiteTypes().split(",");
List<String> channelList =getChannelList(channel);
String[] channel = queryRequest.getSiteTypes().split(",");
List<String> channelList = getChannelList(channel);
if (TObjectUtils.isListEmpty(channelList)) {
return null;
}
@ -279,18 +369,26 @@ public class TopicQueryService {
/**
* 查询渠道
*/
private List<String> getChannelList(String [] channel){
private List<String> getChannelList(String[] channel) {
List<String> channels = new ArrayList<>();
Map<String,String> map = new HashMap<>();
map.put("0","社交媒体");
map.put("1","新闻资讯");
map.put("2","博客智库");
map.put("3","论坛贴吧");
map.put("4","网络视频");
map.put("5","电商网站");
map.put("6","搜索引擎");
map.put("7","生活方式");
for (String id:channel) {
Map<String, String> map = new HashMap<>();
// map.put("0","社交媒体");
// map.put("1","新闻资讯");
// map.put("2","博客智库");
// map.put("3","论坛贴吧");
// map.put("4","网络视频");
// map.put("5","电商网站");
// map.put("6","搜索引擎");
// map.put("7","生活方式");
map.put("0", "社交媒体");
map.put("1", "政企官网");
map.put("2", "智库资讯");
map.put("3", "新闻媒体");
map.put("4", "网络视频");
map.put("5", "电商网站");
map.put("6", "搜索引擎");
map.put("7", "内容分享");
for (String id : channel) {
channels.add(map.get(id));
}
return channels;

4
cl_search_api/src/main/java/com/bfd/mf/common/service/common/CrudService.java

@ -1,12 +1,10 @@
package com.bfd.mf.common.service.common;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.web.entity.mysql.Model;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import org.springframework.data.repository.CrudRepository;
import java.io.Serializable;
import java.util.*;
import java.util.Optional;
public abstract class CrudService<M extends Model, R extends CrudRepository<M, Long>> implements Serializable {

4
cl_search_api/src/main/java/com/bfd/mf/common/service/es/ClusterService.java

@ -24,7 +24,9 @@ public class ClusterService extends CrudService<Cluster, ClusterRepository> {
* 只对应获取 mini && normal cluster
*/
public Cluster findClusterByType(int type) {
if (type == Cluster.CLUSTER_TYPE.mini_cluster_type || type == Cluster.CLUSTER_TYPE.normal_cluster_type) {
if (type == Cluster.CLUSTER_TYPE.mini_cluster_type
|| type == Cluster.CLUSTER_TYPE.normal_cluster_type
|| type == Cluster.CLUSTER_TYPE.special_cluster_type) {
return repo.findClusterByType(type);
}
return null;

7
cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsCommonService.java

@ -1,15 +1,16 @@
package com.bfd.mf.common.service.es;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.util.ESServerUtils;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.nlp.common.util.string.TStringUtils;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.List;
@Service
public class EsCommonService {

2
cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorCountService.java

@ -8,7 +8,6 @@ import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.config.BFDApiConfig;
import com.bfd.mf.service.SearchAuthorService;
import com.bfd.nlp.common.util.string.TStringUtils;
import org.apache.commons.collections.map.HashedMap;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
@ -17,7 +16,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.*;
@Service
public class EsQueryAuthorCountService {

23
cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryAuthorService.java

@ -9,14 +9,20 @@ import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.config.BFDApiConfig;
import com.bfd.mf.service.SearchAuthorService;
import com.bfd.nlp.common.util.string.TStringUtils;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class EsQueryAuthorService {
@ -68,6 +74,7 @@ public class EsQueryAuthorService {
siteIconMap.put(map.get("cid").toString().toLowerCase(), map.get("site_icon").toString());
}
}
List<JSONObject> newResult = new ArrayList<>();
for (JSONObject json: result) {
JSONObject newJson= json;
@ -96,7 +103,8 @@ public class EsQueryAuthorService {
BoolQueryBuilder boolQueryBuilder = null;
boolQueryBuilder = topicQueryService.queryByConditions_v1(queryRequest);
// 二次查询 关键词不为空就添加关键词查询语句 = 0 content 1 title 2 author 3 con+tit 4con+aut 5con+com
Integer searchType = queryRequest.getSearchType(); // 单选 0:主贴;1:评论;2:用户
// 单选 0:主贴;1:评论;2:用户
Integer searchType = queryRequest.getSearchType();
// String searchScope = queryRequest.getSearchScope(); //复选 0:标题;1:正文;2:作者 多个用,分割 0,1
String keyword = queryRequest.getKeyword();
@ -104,17 +112,20 @@ public class EsQueryAuthorService {
boolQueryBuilder.filter(searchTextBuilder);
// Map<String ,Float> fields = new HashedMap();
if (TStringUtils.isNotEmpty(keyword)) {
if(searchType == 0){ // 主贴的话 标题和内容
// 主贴的话 标题和内容
if(searchType == 0){
MatchPhraseQueryBuilder titleQuery = QueryBuilders.matchPhraseQuery(ESConstant.TITLE, keyword).slop(0);
MatchPhraseQueryBuilder contentQuery = QueryBuilders.matchPhraseQuery(ESConstant.CONTENT, keyword).slop(0);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().should(titleQuery).should(contentQuery);
bqb.must(queryBuilder);
}else if (searchType == 1){ // 评论的话 评论内容
// 评论的话 评论内容
}else if (searchType == 1){
// MatchPhraseQueryBuilder contentQuery = QueryBuilders.matchPhraseQuery(ESConstant.CONTENT, keyword).slop(0);
// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(contentQuery);
// qb.must(queryBuilder);
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(ESConstant.CONTENT, keyword).slop(0));
}else if (searchType == 2){ // 用户 就只查 用户名
// 用户 就只查 用户名
}else if (searchType == 2){
boolQueryBuilder.must(QueryBuilders.queryStringQuery("*"+keyword+"*").field(ESConstant.AUTHOR));
}
}

75
cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQMini.java

@ -11,7 +11,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
@Service
public class EsQueryServiceForSQMini {
@ -28,7 +29,8 @@ public class EsQueryServiceForSQMini {
// 注册数据查询来源
clusterName = bfdApiConfig.esMiniName();
String sourceAddress [] = bfdApiConfig.esMiniAddress();
EsUtils.registerCluster(clusterName, sourceAddress);// 配置文件中的 es-source
// 配置文件中的 es-source
EsUtils.registerCluster(clusterName, sourceAddress);
}
@ -38,11 +40,16 @@ public class EsQueryServiceForSQMini {
BoolQueryBuilder boolQueryBuilder = getQueryBuilder.getQueryBuilder(queryRequest);
//logger.info("[EsQueryService] queryDataFromOneSubject: indexName = " + indexName[0] + "; qb:\n" + "{}.", boolQueryBuilder.toString());
logger.info("[EsQueryService] queryDataFromOneSubject: indexName : {}" ,indexName[0]);
Integer limit = queryRequest.getLimit(); //每页的数量
Integer start = (queryRequest.getPage() - 1) * limit; //起始页(0,20,40....)
String orderFlag = queryRequest.getOrder(); // 排序方式 asc/desc
String sortFlag = queryRequest.getSidx(); // 排序字段
Integer searchType = queryRequest.getSearchType(); // 主贴评论还是用户
//每页的数量
Integer limit = queryRequest.getLimit();
//起始页(0,20,40....)
Integer start = (queryRequest.getPage() - 1) * limit;
// 排序方式 asc/desc
String orderFlag = queryRequest.getOrder();
// 排序字段
String sortFlag = queryRequest.getSidx();
// 主贴评论还是用户
Integer searchType = queryRequest.getSearchType();
List<JSONObject> result = EsUtils.query(clusterName, indexName, boolQueryBuilder, sortFlag, orderFlag, limit, start, searchType);
return result;
}catch (Exception e){
@ -50,26 +57,39 @@ public class EsQueryServiceForSQMini {
}
}
/**
* 查询数据量
* @param indexName
* @param queryRequest
* @return
*/
public Long queryDataCountFromOneSubject(String[] indexName, QueryRequest queryRequest) {
try {
logger.debug("[EsQueryServiceForSQMini - 专题] queryDataCountFromOneSubject ...");
BoolQueryBuilder boolQueryBuilder = getQueryBuilder.getQueryBuilder(queryRequest);
Integer searchType = queryRequest.getSearchType();
Integer size = queryRequest.getLimit();
//logger.info("[EsQueryService] queryDataCountFromOneSubject indexName = " + indexName + "; qb:" + "{}.", boolQueryBuilder.toString());
Long totalCount = EsUtils.queryTotalCount(clusterName, indexName, boolQueryBuilder, searchType );
Long totalCount = EsUtils.queryTotalCountNew(clusterName, indexName, boolQueryBuilder, searchType );
//System.out.println("EsQueryServiceForSQMini : queryDataCountFromOneSubject " + totalCount);
return totalCount;
}catch (Exception e){
return 0L;
}
}
/**
* 复制示例专题的数据到新的专题
* @param indexName
* @param queryRequest
* @return
*/
public JSONObject exportDataFromOneSubject(String[] indexName, QueryRequest queryRequest) {
try {
logger.info("[EsQueryServiceForSQMini - 专题] exportDataFromOneSubject start ...");
BoolQueryBuilder boolQueryBuilder =getQueryBuilder.getQueryBuilder(queryRequest);
// logger.info("[EsQueryServiceForSQMini-专题] queryDataFromOneSubject: indexName = " + indexName + "; qb:" + "{}.", boolQueryBuilder.toString());
Integer limit = queryRequest.getLimit(); //每页的数量
//每页的数量
Integer limit = queryRequest.getLimit();
String scrollId = queryRequest.getScrollId();
Integer searchType = queryRequest.getSearchType();
JSONObject result = EsUtils.queryForExport(clusterName, indexName, boolQueryBuilder, limit,scrollId,searchType);
@ -80,6 +100,41 @@ public class EsQueryServiceForSQMini {
}
}
public long reIndexData(String indexList, String newIndex) {
try{
long created = EsUtils.reIndex(clusterName,indexList,newIndex);
return created;
}catch (Exception e){
e.printStackTrace();
return 0;
}
}
public void deleteBySubjectId(String indexName) {
try{
EsUtils.delIndex(clusterName,indexName);
}catch (Exception e){
e.printStackTrace();
}
}
public void deleteBySubjectIdByCid(String indexName, String cid) {
try{
EsUtils.delIndexByCid(clusterName,indexName,cid);
}catch (Exception e){
e.printStackTrace();
}
}
public void deleteBySubjectIdByCrawlDataFlag(String indexName, String cid, String crawlDataFlag) {
try{
EsUtils.delIndexByCrawlDataFlag(clusterName,indexName,cid,crawlDataFlag);
}catch (Exception e){
e.printStackTrace();
}
}
// public JSONObject exportDataFromOneSubjectTestGroupBy(String[] indexName, QueryRequest queryRequest) {
// try {
// logger.info("[exportDataFromOneSubjectTestGroupBy - 专题] exportDataFromOneSubject start ...");

2
cl_search_api/src/main/java/com/bfd/mf/common/service/es/EsQueryServiceForSQNormal.java

@ -11,7 +11,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.List;
@Service
public class EsQueryServiceForSQNormal {

12
cl_search_api/src/main/java/com/bfd/mf/common/service/es/GetQueryBuilder.java

@ -4,8 +4,11 @@ import com.bfd.mf.common.service.cache.TopicQueryService;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.nlp.common.util.string.TStringUtils;
import org.apache.commons.collections.map.HashedMap;
import org.elasticsearch.index.query.*;
import org.apache.commons.collections4.map.HashedMap;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.BucketOrder;
@ -32,10 +35,12 @@ public class GetQueryBuilder {
// 如果要根据ID 查询数据 如果查ID 后面的条件就不用查了
if(null != queryRequest.getDataIds() && !("").equals(queryRequest.getDataIds())) {
String taskId = queryRequest.getTaskId();
String dataIds = queryRequest.getDataIds();
List<String> dataIdList = getDataIdList(dataIds);
QueryBuilder queryBuilder = QueryBuilders.termsQuery(ESConstant.DATA_ID, dataIdList);
boolQueryBuilder = QueryBuilders.boolQuery().filter(queryBuilder);
boolQueryBuilder = QueryBuilders.boolQuery().filter(queryBuilder)
.must(QueryBuilders.termQuery("taskId",taskId));
qb.must(boolQueryBuilder);
return qb;
}
@ -60,7 +65,6 @@ public class GetQueryBuilder {
boolQueryBuilder.must(QueryBuilders.queryStringQuery("*"+keyword+"*").field(ESConstant.AUTHOR));
}
}
qb.must(boolQueryBuilder);
return qb;
}

2
cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java

@ -64,7 +64,7 @@ public class ParseSearchScopeService {
.must(QueryBuilders.termQuery(ESConstant.DOC_TYPE,ESConstant.ITEM)));
} else if(searchType == 2){
searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(ESConstant.PRIMARY, 2))
.must(QueryBuilders.termsQuery(ESConstant.PAGETYPE,"userInfoPage","newsuser"));
.must(QueryBuilders.termsQuery(ESConstant.PAGETYPE,"userInfoPage","newsuser","videoAccount"));
}
return searchScopeQuery;
}

5
cl_search_api/src/main/java/com/bfd/mf/common/service/es/SubjectQueryDataService.java

@ -12,7 +12,10 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

2
cl_search_api/src/main/java/com/bfd/mf/common/service/text/TextService.java

@ -2,7 +2,6 @@ package com.bfd.mf.common.service.text;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity;
import com.bfd.mf.config.BFDApiConfig;
import net.logstash.logback.encoder.org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang3.StringUtils;
@ -27,7 +26,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

4
cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java

@ -1,11 +1,10 @@
package com.bfd.mf.common.util.analysis;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.util.enums.SpreadEnums;
import com.bfd.mf.common.util.utility.DateUtil;
import com.bfd.mf.common.web.vo.view.analysis.*;
import com.bfd.mf.common.web.vo.view.analysis.DataCount;
import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity;
import com.bfd.nlp.common.util.object.TObjectUtils;
import org.slf4j.Logger;
@ -14,7 +13,6 @@ import org.slf4j.LoggerFactory;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.IntStream;
public class DataAnalysisUtils {

1
cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java

@ -10,7 +10,6 @@ import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class ResponseParseUtils {
private static Logger logger = LoggerFactory.getLogger(ResponseParseUtils.class);

13
cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java

@ -265,11 +265,13 @@ public class ESConstant {
/**
* SQ 4.0新增 UGC EGC PGC
*/
public static final String SEARCH_SCOPE_UGC = "ugc";
// public static final String SEARCH_SCOPE_UGC = "ugc";
// public static final String SEARCH_SCOPE_EGC = "egc";
// public static final String SEARCH_SCOPE_PGC = "pgc";
public static final String SEARCH_SCOPE_EGC = "egc";
public static final String SEARCH_SCOPE_PGC = "pgc";
public static final String HAS_IMAGE = "hasImage";
public static final String HAS_VIDEO = "hasVideo";
public static final String HAS_FILE = "hasFile";
/**
* 关键词
*/
@ -365,6 +367,7 @@ public class ESConstant {
public static String CONTENT_SIMHASH = "contentSimHash";
public static String QUOTE_COUNT = "quoteCount";
public static String COLLE_CTCOUNT = "collectCount";
public static String VIEW_CNT = "viewCnt";
// 左侧检索内容那块要显示的数字
public static String CONTENTCOUNT = "contentCount";
public static String COMMENTCOUNT = "commentCount";
@ -775,7 +778,7 @@ public class ESConstant {
ESConstant.PUBTIME , // 发表时间
ESConstant.QUOTE_COUNT , // 转发数
// ESConstant.COMMENTS_COUNT , // 评论数
ESConstant.COMMENTS_COUNT , // 评论数
ESConstant.ATTITUDES_COUNT , // 点赞数
ESConstant.PRICE, // 价格
ESConstant.POST_COUNT, // 销量

261
cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java

@ -3,6 +3,10 @@ package com.bfd.mf.common.util.es;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.constants.ESConstant;
import com.google.common.collect.Maps;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
@ -20,11 +24,16 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.*;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality;
import org.elasticsearch.search.collapse.CollapseBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
@ -32,9 +41,10 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import java.net.InetAddress;
import java.util.*;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public abstract class EsUtils {
@ -75,24 +85,62 @@ public abstract class EsUtils {
Integer size, Integer from,
Integer searchType) {
TransportClient client = getClient(clusterName);
boolean options = true;
boolean optionsf = false;
// CollapseBuilder collapseBuilder = new CollapseBuilder(ESConstant.DATA_ID + ".keyword");
// if(searchType == 0 || searchType == 2) { // 表示查的是主贴电商数据要根据 docId 消重才行
// collapseBuilder = new CollapseBuilder(ESConstant.DOC_ID);
// }
// from +size 分页 查询方式
// 现在不同任务的同一条数据不做消重因此同一个DOCID 的数据会有多条因此只有查主贴的时候需要用DOCID 消重
CollapseBuilder collapseBuilder = new CollapseBuilder(ESConstant.DATA_ID);
if (searchType == 0) {
collapseBuilder = new CollapseBuilder(ESConstant.DOC_ID);
}
// from + size 分页 查询方式
SearchRequestBuilder requestBuilder = client.prepareSearch().setIndices(index)
.setIndicesOptions(IndicesOptions.fromOptions(options, options, options, optionsf))
.addSort(sortFlag, orderFlag.equals(ESConstant.ASC) ? SortOrder.ASC : SortOrder.DESC)
.setQuery(queryBuilder)
// .setCollapse(collapseBuilder)
.setCollapse(collapseBuilder)
.setSize(size)
.setFrom(from);
System.out.println(requestBuilder);
SearchResponse searchResponse = requestBuilder.execute().actionGet();
// List<JSONObject> dataList = Lists.newLinkedList();
List<JSONObject> dataList = new ArrayList<>();
if (searchResponse.getHits().totalHits > 0) {
SearchHit[] hits = searchResponse.getHits().getHits();
for (int i = 0; i < hits.length; i++) {
JSONObject data = new JSONObject();
data.putAll(hits[i].getSourceAsMap());
data.put("subjectId", hits[i].getIndex()
.replace("cl_major_", "")
.replace("cl_subject_", "")
.replace("cl_special_1.0_", ""));
dataList.add(data);
}
}
return dataList;
}
public static List<JSONObject> queryNew(String clusterName, String[] index,
final QueryBuilder queryBuilder,
String sortFlag, String orderFlag,
Integer size, Integer from,
Integer searchType) {
TransportClient client = getClient(clusterName);
boolean options = true;
boolean optionsf = false;
// 现在不同任务的同一条数据不做消重因此同一个DOCID 的数据会有多条因此只有查主贴的时候需要用DOCID 消重
CollapseBuilder collapseBuilder = new CollapseBuilder(ESConstant.DATA_ID);
if (searchType == 0) {
collapseBuilder = new CollapseBuilder(ESConstant.DOC_ID);
}
// from + size 分页 查询方式
SearchRequestBuilder requestBuilder = client.prepareSearch().setIndices(index)
.setIndicesOptions(IndicesOptions.fromOptions(options, options, options, optionsf))
.addSort(sortFlag, orderFlag.equals(ESConstant.ASC) ? SortOrder.ASC : SortOrder.DESC)
.setQuery(queryBuilder)
.setCollapse(collapseBuilder)
.setSize(size)
.setFrom(from);
System.out.println(requestBuilder);
SearchResponse searchResponse = requestBuilder.execute().actionGet();
@ -103,29 +151,41 @@ public abstract class EsUtils {
for (int i = 0; i < hits.length; i++) {
JSONObject data = new JSONObject();
data.putAll(hits[i].getSourceAsMap());
data.put("subjectId",hits[i].getIndex().replace("cl_major_","").replace("cl_subject_",""));
data.put("subjectId", hits[i].getIndex()
.replace("cl_major_", "")
.replace("cl_subject_", "")
.replace("cl_special_1.0_", ""));
dataList.add(data);
}
}
return dataList;
}
/**
* 查询数据量 count
* @param clusterName
* @param index
* @param queryBuilder
* @param searchType
* @return
*/
public static Long queryTotalCount(String clusterName, String[] index,
QueryBuilder queryBuilder,
Integer searchType) {
TransportClient client = getClient(clusterName);
Map<String, String> settings = new HashMap<>();
settings.put("index.max_result_window", "200000000");
updSetting(index[0], settings, client);
boolean options = true;
boolean optionsf = false;
// CollapseBuilder collapseBuilder = new CollapseBuilder(ESConstant.DATA_ID+".keyword");
// if(searchType == 0) { // 表示查的是主贴电商数据要根据 docId 消重才行
// collapseBuilder = new CollapseBuilder(ESConstant.DOC_ID);
// }
int querysize = 400000;
// 现在不通任务的同一条数据不做消重因此同一个DOCID 的数据会有多条因此只有查主贴的时候需要用 DOCID 消重
CollapseBuilder collapseBuilder = new CollapseBuilder(ESConstant.DATA_ID);
if (searchType == 0) {
collapseBuilder = new CollapseBuilder(ESConstant.DOC_ID);
}
try {
TransportClient client = getClient(clusterName);
// JSONObject settings = JSONObject.parseObject("{\"max_result_window\": \"200000000\"}");
Map<String,String> settings = new HashMap<>();
settings.put("index.max_result_window","200000000");
updSetting(index[0],settings,client);
SearchResponse response = null;
// if(searchType == 0){ // 查的是主贴需要用 limit 取数不然 默认limit =10
// response = client.prepareSearch().setIndices(index)
@ -138,14 +198,14 @@ public abstract class EsUtils {
response = client.prepareSearch().setIndices(index)
.setIndicesOptions(IndicesOptions.fromOptions(options, options, options, optionsf))
.setQuery(queryBuilder)
// .setCollapse(collapseBuilder)
.setCollapse(collapseBuilder)
.setSize(1000)
.execute().actionGet();
// }
Long size = response.getHits().getTotalHits();
logger.info("[queryTotalCount] 聚合前的总量: {} 这个是聚合前的数据量 ",size);
Long size = response.getHits().getTotalHits();
logger.info("[queryTotalCount] 聚合前的总量: {} 这个是聚合前的数据量 ", size);
Long aggrSize = Long.valueOf(response.getHits().getHits().length);
logger.info("[queryTotalCount] 聚合后的数据量: {} ",aggrSize);
logger.info("[queryTotalCount] 聚合后的数据量: {} ", aggrSize);
// 当聚合后的结果为 10000 或者 是评论数据的时候就不用聚合了
// if(aggrSize == 10000 || searchType == 1 || searchType == 2){
// size = size;
@ -153,21 +213,49 @@ public abstract class EsUtils {
// size = aggrSize;
// }
return size;
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
}
return 0L;
}
public static Long queryTotalCountNew(String clusterName, String[] index,
QueryBuilder queryBuilder,
Integer searchType) {
TransportClient client = getClient(clusterName);
boolean options = true;
boolean optionsf = false;
// 现在不同任务的同一条数据不做消重因此同一个DOCID 的数据会有多条因此只有查主贴的时候需要用DOCID 消重
String count = "count";
AggregationBuilder aggregation ;
if(searchType == 0) {
aggregation = AggregationBuilders.cardinality(count).field(ESConstant.DOC_ID);
}else{
aggregation = AggregationBuilders.cardinality(count).field(ESConstant.DATA_ID);
}
//searchSourceBuilder.aggregation(aggregation);
// from + size 分页 查询方式
SearchRequestBuilder requestBuilder = client.prepareSearch().setIndices(index)
.setIndicesOptions(IndicesOptions.fromOptions(options, options, options, optionsf))
.setQuery(queryBuilder)
.addAggregation(aggregation);
// System.out.println(requestBuilder);
Aggregations aggregations = requestBuilder.get().getAggregations();
Cardinality cardinality = aggregations.get(count);
// System.out.println("1111 : "+cardinality.getValue());
return cardinality.getValue();
}
public static JSONObject queryForExport(String clusterName,
String indexName [],
String indexName[],
BoolQueryBuilder boolQueryBuilder,
Integer limit,
String scrollId,
Integer searchType) {
Map<String,Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>();
TransportClient client = getClient(clusterName);
SearchResponse searchResponse = null;
try {
@ -181,9 +269,9 @@ public abstract class EsUtils {
.setScroll(new TimeValue(300000))
.execute()
.actionGet();//注意:首次搜索并不包含数据
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
logger.error(" queryForExport 首次 查询报错!!!IndexName : {} " , indexName[0]);
logger.error(" queryForExport 首次 查询报错!!!IndexName : {} ", indexName[0]);
}
} else {
System.out.println("翻页查询");
@ -200,20 +288,22 @@ public abstract class EsUtils {
}
//获取总数量
long totalCount = searchResponse.getHits().getTotalHits();
int page=(int)totalCount/( 2 * limit);//计算总页数,每次搜索数量为分片数*设置的size大小
//计算总页数,每次搜索数量为分片数*设置的size大小
int page = (int) totalCount / (2 * limit);
//int page = 2 * limit;
// System.out.println("queryForExport : " + totalCount + " ; page = " + page + " ; scrollId = " + searchResponse.getScrollId());
// System.out.println("queryForExport : " + totalCount + " ; page = " + page + " ; scrollId = " + searchResponse.getScrollId());
for (int i = 0; i <= page; i++) {
//再次发送请求,并使用上次搜索结果的ScrollId
List<JSONObject> monitorLists = parseSearchResponse(searchResponse);
if(monitorLists.size() == 0){ // 回收 scrollId
boolean clear = clearScroll(client,scrollId);
// 回收 scrollId
if (monitorLists.size() == 0) {
boolean clear = clearScroll(client, scrollId);
}
result.put(ESConstant.MONITORLISTS, monitorLists);
}
result.put(ESConstant.ALLDOCNUMBER, totalCount);
result.put(ESConstant.SCROLLID, searchResponse.getScrollId());
}catch (Exception e){
} catch (Exception e) {
logger.info("[EsUtils] queryForExport error !!! ");
result.put(ESConstant.MONITORLISTS, new ArrayList<JSONObject>());
result.put(ESConstant.ALLDOCNUMBER, 0L);
@ -225,7 +315,7 @@ public abstract class EsUtils {
/**
* 不在使用滚动时清除id
*/
public static boolean clearScroll( TransportClient client ,String scrollId) {
public static boolean clearScroll(TransportClient client, String scrollId) {
ClearScrollRequestBuilder clearScrollRequestBuilder = client.prepareClearScroll();
clearScrollRequestBuilder.addScrollId(scrollId);
ClearScrollResponse response = clearScrollRequestBuilder.get();
@ -240,7 +330,7 @@ public abstract class EsUtils {
for (SearchHit searchHit : hits.getHits()) {
try {
JSONObject json = JSONObject.parseObject(searchHit.getSourceAsString());
json.put("subjectId",searchHit.getIndex());
json.put("subjectId", searchHit.getIndex());
result.add(json);
// WriteMethod.writeMethod("dataIds",searchHit.getSourceAsMap().get("dataId").toString());
} catch (Exception e) {
@ -251,28 +341,28 @@ public abstract class EsUtils {
}
// 封装方法bai
public static boolean updSetting(String index, Map settings,TransportClient client) {
public static boolean updSetting(String index, Map settings, TransportClient client) {
// 先判断索引是否存在
boolean boo = isExistsIndex(index,client);
if(boo) {
boolean boo = isExistsIndex(index, client);
if (boo) {
UpdateSettingsRequest request = Requests.updateSettingsRequest(index).settings(settings);
UpdateSettingsResponse response = client.admin().indices().updateSettings(request).actionGet();
return response.isAcknowledged();
}else{
} else {
return false;
}
}
private static boolean isExistsIndex(String indexName,TransportClient client){
IndicesExistsResponse response = client.admin().indices().exists(
private static boolean isExistsIndex(String indexName, TransportClient client) {
IndicesExistsResponse response = client.admin().indices().exists(
new IndicesExistsRequest().indices(indexName)).actionGet();
return response.isExists();
}
public static Terms queryByTag(String clusterName, String index,String filter,
public static Terms queryByTag(String clusterName, String index, String filter,
QueryBuilder queryBuilder,
AggregationBuilder aggregationBuilder){
AggregationBuilder aggregationBuilder) {
TransportClient client = getClient(clusterName);
SearchRequestBuilder requestBuilder = client.prepareSearch(index)
@ -280,9 +370,9 @@ public abstract class EsUtils {
.setQuery(queryBuilder)
.addAggregation(aggregationBuilder);
logger.info("requestBuilder: " , requestBuilder.toString().replace("\n","").replace("\r","").replace(" ",""));
logger.info("requestBuilder: ", requestBuilder.toString().replace("\n", "").replace("\r", "").replace(" ", ""));
SearchResponse response = requestBuilder.execute().actionGet();
Terms aggregation = response.getAggregations().get(filter+"Tag");
Terms aggregation = response.getAggregations().get(filter + "Tag");
return aggregation;
}
@ -298,18 +388,18 @@ public abstract class EsUtils {
.actionGet();//注意:首次搜索并不包含数据
} catch (Exception e) {
e.printStackTrace();
logger.error(" queryForExport 首次 查询报错!!!IndexName : {} " ,indexName[0]);
logger.error(" queryForExport 首次 查询报错!!!IndexName : {} ", indexName[0]);
}
//再次发送请求,并使用上次搜索结果的ScrollId
List<JSONObject> monitorLists = parseSearchResponse(searchResponse);
if(monitorLists.size() >0){
if (monitorLists.size() > 0) {
return monitorLists.get(0);
}else{
} else {
return new JSONObject();
}
}
public static long updateByDocId(String clusterName, String index,String script,
public static long updateByDocId(String clusterName, String index, String script,
String docId, Map<String, Object> params) {
TransportClient client = getClient(clusterName);
try {
@ -323,9 +413,74 @@ public abstract class EsUtils {
updateRequest.setRefreshPolicy("true");
UpdateResponse response = client.update(updateRequest).get();
return response.getVersion();
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
return 0L;
}
}
public static long reIndex(String clusterName, String indexList, String newIndex) {
try {
TransportClient client = getClient(clusterName);
System.out.println(indexList + " *** " + newIndex);
ReindexRequestBuilder builder = ReindexAction.INSTANCE
.newRequestBuilder(client)
.source(indexList)
.destination(newIndex);
String newAliex = newIndex.replace("cl_special_1.0_", "cl_major_");
BulkByScrollResponse response = builder.get();
IndicesAliasesRequestBuilder indicesBuilder = IndicesAliasesAction.INSTANCE.newRequestBuilder(client).addAlias(newIndex, newAliex);
IndicesAliasesResponse IndicesResponse = indicesBuilder.get();
System.out.println("******* : " + response);
System.out.println("##### : " + IndicesResponse);
return response.getCreated();
} catch (Exception e) {
e.printStackTrace();
return 0;
}
}
public static void delIndex(String clusterName, String index) {
try {
TransportClient client = getClient(clusterName);
System.out.println("**** " + index);
DeleteIndexResponse response = client.admin().indices().prepareDelete(index).get();
System.out.println(response.isAcknowledged());
} catch (Exception e) {
e.printStackTrace();
}
}
public static void delIndexByCid(String clusterName, String indexName, String cid) {
try {
TransportClient client = getClient(clusterName);
System.out.println("**** " + indexName);
cid = cid.toLowerCase();
DeleteByQueryRequestBuilder builder = DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
.filter(QueryBuilders.matchQuery("enSource", cid)).source(indexName);
BulkByScrollResponse response2 = builder.get();
long deleted = response2.getDeleted();
System.out.println(deleted);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void delIndexByCrawlDataFlag(String clusterName, String indexName, String cid, String crawlDataFlag) {
try {
TransportClient client = getClient(clusterName);
System.out.println("**** " + indexName);
cid = cid.toLowerCase();
DeleteByQueryRequestBuilder builder = DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
.filter(QueryBuilders.termQuery(ESConstant.EN_SOURCE, cid))
.filter(QueryBuilders.termQuery(ESConstant.CRAWLDATAFLAG, crawlDataFlag))
.source(indexName);
BulkByScrollResponse response2 = builder.get();
long deleted = response2.getDeleted();
System.out.println(deleted);
} catch (Exception e) {
e.printStackTrace();
}
}
}

2
cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils2.java

@ -4,7 +4,6 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.RateLimiter;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
@ -33,7 +32,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;
import javax.annotation.PostConstruct;
import java.net.InetAddress;
import java.util.List;
import java.util.Map;

3
cl_search_api/src/main/java/com/bfd/mf/common/util/es/MonitorUtils.java

@ -8,10 +8,7 @@ import org.elasticsearch.search.SearchHits;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class MonitorUtils {

10
cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java

@ -7,7 +7,6 @@ import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity;
import com.bfd.nlp.common.util.object.TObjectUtils;
import com.bfd.nlp.common.util.string.TStringUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
@ -16,7 +15,10 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@Component
@ -129,11 +131,11 @@ public class SliceScrollUtil {
List<String> currentIndexList = null;
if(null != queryRequest.getSubjectId() && !("all").equals(queryRequest.getSubjectId())){
logger.info("查询 【专题数据】 subjectId :{}" ,queryRequest.getSubjectId());
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
}else{
logger.info("[SliceScrollUtil] fetchResultSubjectCache : 查询 【全局数据】");
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
currentIndexList = subjectQueryDataService.getIndexListByTimeRange(cluster, queryRequest.getStartTime(),queryRequest.getEndTime());
}

2
cl_search_api/src/main/java/com/bfd/mf/common/util/utility/DateUtil.java

@ -19,13 +19,11 @@ package com.bfd.mf.common.util.utility;
import com.bfd.nlp.common.util.string.TStringUtils;
import java.io.UnsupportedEncodingException;
import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.TimeZone;
public class DateUtil {

46
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/cache/Cluster.java

@ -22,47 +22,37 @@ public class Cluster extends Model implements Serializable, Comparable<Cluster>
@Override
public int compareTo(Cluster o) {
if (this.getShardNum() >= o.getShardNum())
if (this.getShardNum() >= o.getShardNum()) {
return -1;
else
}else {
return 1;
}
}
public static final class CLUSTER_TYPE {
/**
* 日期索引
*/
public static final int normal_cluster_type = 0; // 172.18.1.134:9201 的集群 日期索引 SQ_Normal
/**
* 日期索引
*/
public static final int newnormal_cluster_type = 1; // 172.18.1.134:9201 的集群 日期索引 SQ_Normal_2
/**
* 小集群
* 专题索引
*/
public static final int mini_cluster_type = 2; // 111 的ES 集群 专题索引 SQ_Mini
public static final int mini_cluster_type = 2; // 172.18.1.147:9213 的ES 集群 专题索引 SQ_Mini
/**
* 大集群
* 专题索引
*/
public static final int normal_cluster_type = 0; // 109 的集群 日期索引 SQ_Normal
// /**
// * 缓存
// */
// // public static final int subject_cluster_type = 2; //
//
// /**
// * 微博转发
// */
// public static final int weibo_cluster_type = 4; // 147的ES 集群 前台打标后SQ_Mini
//
// public static final int reply_cluster_type = 7; // 已废弃 原160的ES
// /**
// * 2期数据分析 集群
// */
// public static final int data_analysis_cluster_type = 5; // 158 的ES 集群后台打标后的数据 SQ_Reputation_new
// /**
// * 对应索引以 cl_company_*
// */
// public static final int data_company_analysis_cluster_type = 6; // 已废弃 原160的ES
public static final int special_cluster_type = 4; // 172.18.1.81:9201 的ES 集群 专题索引 SQ_Mini_2
}
public static Long getClusterIdByType(SubjectCluster subjectCluster, int type) {
if (type == Cluster.CLUSTER_TYPE.mini_cluster_type) {
if (type == CLUSTER_TYPE.mini_cluster_type) {
return subjectCluster.getMiniClusterId();
} else if (type == Cluster.CLUSTER_TYPE.normal_cluster_type) {
} else if (type == CLUSTER_TYPE.normal_cluster_type) {
return subjectCluster.getNormalClusterId();
// } else if (type == Cluster.CLUSTER_TYPE.subject_cluster_type) {
// return subjectCluster.getClusterId();

1
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/topic/ParseExcelTask.java

@ -1,6 +1,5 @@
package com.bfd.mf.common.web.entity.mysql.topic;
import com.bfd.mf.common.web.entity.mysql.Model;
import org.joda.time.DateTime;
import javax.persistence.*;

469
cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/UserRepository.java

@ -1,469 +0,0 @@
///*
// * Copyright (C) 2016 Baifendian Corporation
// * <p>
// * Licensed under the Apache License, Version 2.0 (the "License");
// * you may not use this file except in compliance with the License.
// * You may obtain a copy of the License at
// * <p>
// * http://www.apache.org/licenses/LICENSE-2.0
// * <p>
// * Unless required by applicable law or agreed to in writing, software
// * distributed under the License is distributed on an "AS IS" BASIS,
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// * See the License for the specific language governing permissions and
// * limitations under the License.
// */
//
//package com.bfd.mf.common.web.repository.mysql;
//
//import com.bfd.mf.common.web.entity.mysql.User;
//import org.springframework.data.jpa.repository.Query;
//import org.springframework.data.repository.PagingAndSortingRepository;
//import org.springframework.data.repository.query.Param;
//import org.springframework.stereotype.Repository;
//
//@Repository
//public interface UserRepository extends PagingAndSortingRepository<User, Long> {
//
//// User findByUserNameAndDel(String userName, Boolean del);
////
//// List<User> findByAccountAndDel(String account, Boolean del);
////
////
//// @Query(value = "SELECT * FROM cl_user WHERE del = FALSE AND status = 1 AND account = ?1 ", nativeQuery = true)
//// User findAvailableByAccount(String account);
////
//// @Query(value = "SELECT * FROM cl_user WHERE del = FALSE AND account = ?1 ", nativeQuery = true)
//// User findUserByAccount(String account);
////
//// User findByUserNameAndPasswordAndDel(String userName, String pwd, Boolean del);
////
//// User findByAccountAndPasswordAndDel(String account, String pwd, Boolean del);
////
//// User findByCompanyIdAndUserNameAndDel(Long companyId, String userName, Boolean del);
////
//// User findByCompanyIdAndAccountAndDel(Long companyId, String account, Boolean del);
//
// User findByTokenAndDel(String token, Boolean del);
//
//// @Query(value = "SELECT * FROM cl_user WHERE token = ?1 AND status = ?2", nativeQuery = true)
//// User findByTokenAndStatus(String token, int status);
////
//// User findByTokenAndStatusAndDel(String token, int status, Boolean del);
//
// @Query(value = "SELECT company_id FROM cl_user WHERE token = :token AND del = :del", nativeQuery = true)
// Long getCompanyIdByTokenAndDel(@Param("token") String token, @Param("del") Boolean del);
//
//
//// @Query("select u from User u where u.roleId =?1 or u.roleId like '?1,%' or u.roleId like '%,?1' or u.roleId like " +
//// "'%,?1,%' and del = false")
//// List<User> findByRoleId(String rid);
////
//// List<User> findByStatusAndCompanyIdAndDel(Integer status, Long companyId, Boolean del);
////
//// List<User> findByCompanyIdAndDel(Long companyId, Boolean del);
////
////
//// /**
//// * Get all users with role and creator information
//// *
//// * @param companyId
//// * @return
//// */
//// @Query(value = "SELECT " +
//// "a.id , " +
//// "a.token , " +
//// "a.account , " +
//// "a.userName , " +
//// "r.id AS roleId , " +
//// "r.name as roleName, " +
//// "b.userName AS creator, " +
//// "a.status, " +
//// "a.email, " +
//// "a.wechat, " +
//// "a.wechatId, " +
//// "a.mobile, " +
//// "a.wechatVerified, " +
//// "a.wechatOpenId " +
//// "FROM " +
//// "User a , " +
//// "User b , " +
//// "Role r " +
//// "WHERE a.companyId = b.companyId " +
//// "AND a.companyId = :cid " +
//// "AND a.parentId = b.id " +
//// "AND a.roleId = r.id " +
//// "AND a.userName like :name " +
//// "AND (a.del IS NULL OR a.del = false)" +
//// "AND a.wechat IS NOT NULL " +
//// "AND a.wechatId is NOT NULL " +
//// "AND a.wechat <> '' " +
//// "AND a.wechatId <> '' " +
//// "AND a.wechatVerified in (:scope) " +
//// "order by a.createdTime desc")
//// Page<Object> findClientUserWeixinList(@Param("cid") Long companyId, @Param("name") String filterName, Pageable
//// pageable, @Param("scope") List<String> scope);
////
////
//// /**
//// * Get all users with role and creator information
//// *
//// * @param companyId
//// * @return
//// */
//// @Query(value = "SELECT " +
//// "a.id , " +
//// "a.token , " +
//// "a.account , " +
//// "a.userName , " +
//// "r.id AS roleId , " +
//// "r.name as roleName, " +
//// "b.userName AS creator, " +
//// "a.status, " +
//// "a.email, " +
//// "a.wechat, " +
//// "a.wechatId, " +
//// "a.mobile " +
//// "FROM " +
//// "User a , " +
//// "User b , " +
//// "Role r " +
//// "WHERE a.companyId = b.companyId " +
//// "AND a.companyId = :cid " +
//// "AND a.parentId = b.id " +
//// "AND a.roleId = r.id " +
//// "AND a.userName like :name " +
//// "AND (a.del IS NULL OR a.del = false)" +
//// "AND a.email IS NOT NULL " +
//// "AND a.email <> '' " +
//// "order by a.createdTime desc")
//// Page<Object> findClientUserEmailList(@Param("cid") Long companyId, @Param("name") String filterName, Pageable
//// pageable);
////
//// @Modifying
//// @Query("update User set status = ?2, updatedTime = ?3 where id = ?1")
//// int switchUserStatus(Long userId, Integer status, Long updateTime);
////
//// @Modifying
//// @Query("update User set del = ?2, updatedTime = ?3 where id = ?1")
//// int deleteUser(Long userId, Boolean isDel, Long updateTime);
////
////
////// @Query("SELECT " +
////// "a.id ," +
////// "a.account , " +
////// "a.userName, " +
////// "a.email, " +
////// "a.wechat, " +
////// "a.wechatId, " +
////// "a.mobile, " +
////// "r.id AS userGroupId , " +
////// "r.name as userGroupName, " +
////// "a.status, " +
////// "a.createdTime " +
////// "FROM " +
////// "User a , " +
////// "UserGroup r " +
////// "WHERE a.id = :uid " +
////// "AND a.userGroupId = r.id " +
////// "AND (a.del = :isDel ) " +
////// "order by a.createdTime desc")
////// List<Object[]> fetchUser(@Param("uid") Long userId, @Param("isDel") Boolean isDel);
////
////
//// /**
//// * Check if reach user create limitation, return Y if avaliable
//// * <p>
//// * Return NULL if not available
//// *
//// * @param token
//// * @return
//// */
//// @Query(value = "SELECT \"Y\" AS VALID " +
//// "FROM " +
//// " (SELECT company_user_count AS u_limit " +
//// " FROM cl_company " +
//// " WHERE status = 1 " +
//// " AND id = " +
//// " (SELECT company_id " +
//// " FROM cl_user " +
//// " WHERE token = ?1)) a, " +
//// " (SELECT count(1) AS u_count " +
//// " FROM cl_user " +
//// " WHERE company_id = " +
//// " (SELECT company_id " +
//// " FROM cl_user " +
//// " WHERE token = ?1) " +
//// " AND del = FALSE) b " +
//// "WHERE a.u_limit >= b.u_count", nativeQuery = true)
//// List checkIfUserLimitReached(String token);
////
//// /**
//// * Check if user over dead date and open status, return true if available
//// * <p>
//// * Return false if not available
//// *
//// * @param token
//// * @return
//// */
//// @Query(value =
//// " SELECT \"Y\" AS flag " +
//// " FROM cl_company " +
//// " WHERE status = 1 " +
//// " AND deadline_time < UNIX_TIMESTAMP() " +
//// " AND id = " +
//// " (SELECT company_id " +
//// " FROM cl_user " +
//// " WHERE token = ?1 and del = false and status = 1)", nativeQuery = true)
//// List checkIfUserOverDeadDate(String token);
////
//// /**
//// * 返回话题上限数
//// * <p>
//// * SELECT company_topic_count FROM cl_company
//// * WHERE id=(SELECT company_id FROM cl_user WHERE token = '8FB24E550815BCBA')
//// *
//// * @param token
//// * @return
//// */
//// @Query(value =
//// " SELECT company_topic_count " +
//// " FROM cl_company " +
//// " WHERE id = " +
//// " (SELECT company_id" +
//// " FROM cl_user " +
//// " WHERE token = ?1)", nativeQuery = true)
//// int getCompanyTopicCount(String token);
////
//// /**
//// * 返回创建话题的ids
//// *
//// * @param token
//// * @return
//// */
//// @Query(value =
//// " SELECT topic_ids" +
//// " FROM cl_role " +
//// " WHERE company_id = " +
//// " (SELECT company_id" +
//// " FROM cl_user " +
//// " WHERE token = ?1) ", nativeQuery = true)
//// List<String> getSujectIds(String token);
////
//// /**
//// * 返回客户创建的话题总数的结果
//// * <p>
//// * SELECT COUNT(DISTINCT(label_id)) FROM mf_subject
//// * WHERE label_id IN(
//// * SELECT topic_ids FROM mf_role
//// * WHERE company_id ='14')
//// *
//// * @param labelIds
//// * @return
//// */
//// @Query(value =
//// " SELECT count(1) " +
//// " FROM cl_subject " +
//// " WHERE label_id IN " +
//// " (?1)", nativeQuery = true)
//// int getCompanyResult(Set<String> labelIds);
////
//// @Query("select u from User u, Role r " +
//// "where u.roleId = r.id and u.companyId =:cid and " +
//// "r.isSystem >= (select r2.isSystem from Role r2 where r2.id = :rid) and u.del = false")
//// List<User> getUserListByCompanyAndRole(@Param("cid") Long companyId, @Param("rid") Long roleId);
////
//// @Query("select " +
//// "u.account," +
//// "wu.openId," +
//// "wu.subscribe," +
//// "wu.nickName," +
//// "wu.sex," +
//// "wu.language," +
//// "wu.city," +
//// "wu.province," +
//// "wu.headimgUrl," +
//// "wu.subscribeTime," +
//// "wu.unionId," +
//// "wu.remark," +
//// "wu.grouId," +
//// "wu.tagIdList," +
//// "wu.mfRemark " +
//// "from User u, Role r, WechatBind wb, WechatUser wu " +
//// "where u.roleId = r.id and u.companyId =:cid and u.id = wb.userId and wb.openId = wu.openId and " +
//// "r.isSystem >= (select r2.isSystem from Role r2 where r2.id = :rid) and u.del = false")
//// List getUserInfoListByCompanyAndRole(@Param("cid") Long companyId, @Param("rid") Long roleId);
////
//// @Query(value = "select * from cl_user where id =?1", nativeQuery = true)
//// User getUserById(long creator);
////
//// /**
//// * 返回客户ID
//// *
//// * @param token
//// * @return
//// */
//// @Query(value = "select company_id from cl_user where token =?1", nativeQuery = true)
//// Integer findCompanyIdByToken(String token);
////
//// /**
//// * 返回有效客户
//// *
//// * @param id
//// * @param flag
//// * @return
//// */
//// @Query(value = "SELECT token FROM cl_user WHERE company_id=?1 AND del=?2", nativeQuery = true)
//// List<String> findTokenByCompanyId(Long id, boolean flag);
////
//// @Query(value = "SELECT token FROM cl_user WHERE company_id =?1 AND del=?2 AND STATUS=?3 limit 1", nativeQuery = true)
//// String findEffectTokenByCompanyIdAndDel(Long companyId, boolean del, int status);
////
//// @Query(value =
//// "select company_id " +
//// "from cl_user " +
//// "where token = ?1 and del=?2", nativeQuery = true)
//// Long findCompanyIdByTokenAndDel(String token, boolean del);
////
//// @Query(value = "SELECT account FROM cl_user WHERE email=?1 AND status=?2", nativeQuery = true)
//// String fetchUserOfAccountByReceiverAndStatus(String receiver, int status);
////
//// @Query(value =
//// " SELECT company_keyword_count " +
//// " FROM cl_company " +
//// " WHERE id = " +
//// " (SELECT company_id" +
//// " FROM cl_user " +
//// " WHERE token = ?1)", nativeQuery = true)
//// int getCompanyKeywordCount(String token);
////
//// @Query(value = "SELECT company_exclude_count FROM cl_company WHERE id = (SELECT company_id FROM cl_user WHERE token = ?1)", nativeQuery = true)
//// int getCompanyExcludeCount(String token);
////
//// @Query(value = "SELECT role_id FROM cl_user WHERE token = :token AND del = :del", nativeQuery = true)
//// String getRoleIdByTokenAndDel(@Param("token") String token, @Param("del") Boolean del);
////
//// /**
//// * Get all users with role and creator information
//// *
//// * @param companyId
//// * @return
//// */
//// @Query(value = "SELECT " +
//// "a.id , " +
//// "a.account , " +
//// "a.userName , " +
//// "r.id AS userGroupId , " +
//// "r.name as userGroupName, " +
//// //"b.userName AS creator, " +
//// " (case when a.parentId = 0 then '系统' when m.roleType = 1 then '管理员' else b.account end ) AS creator , " +
//// "a.status, " +
//// "a.email, " +
//// "a.wechat, " +
//// "a.wechatId, " +
//// "a.mobile, " +
//// "a.createdTime, "+
//// "r.roleType "+
//// "FROM " +
//// "User a , " +
//// "User b , " +
//// "UserGroup m , " +
//// "UserGroup r " +
//// "WHERE " +
//// //"a.companyId = b.companyId " +
//// " a.companyId = :cid " +
//// "AND a.userGroupId = r.id " +
//// "AND a.userGroupId = :ugid " +
//// "AND b.userGroupId = m.id " +
//// "AND a.parentId = b.id " +
//// "AND a.isShadow <> :isShadow " +
//// //"AND a.roleId = r.id " +
//// "AND (a.userName like :name or a.account like :name) " +
//// "AND (a.del = :isDel) " +
//// "order by a.createdTime desc" )
//// Page<Object> findClientUserListByGroupId(@Param("cid") Long companyId, @Param("name") String filterName, Pageable pageable, @Param("ugid") Long userGroupId, @Param("isDel") Boolean isDel, @Param("isShadow") Integer isShadow);
////
//// @Query(value = "SELECT * FROM cl_user WHERE account = ?1", nativeQuery = true)
//// User findByAccount(String account);
////
//// @Query(value = "SELECT * FROM cl_user WHERE user_group_id = ?1 and del = ?2", nativeQuery = true)
//// List<User> findByUserGroupId(String userGroupId, Boolean del);
////
//// @Query(value = "SELECT token FROM cl_user WHERE id IN (?1)", nativeQuery = true)
//// List<String> getTokenListByIdList(List<Long> idList);
////
//// @Query(value = "SELECT id FROM cl_user WHERE token IN (?1)", nativeQuery = true)
//// List<Integer> getIdListByTokenList(List<String> tokenList);
////
//// @Query(value = "SELECT " +
//// "a.id , " +
//// "a.account , " +
//// "a.userName , " +
//// "r.id AS userGroupId , " +
//// "r.name as userGroupName, " +
//// // "b.userName AS creator, " +
//// " (case when a.parentId = 0 then '系统' when m.roleType = 1 then '管理员' else b.account end ) AS creator , " +
//// "a.status, " +
//// "a.email, " +
//// "a.wechat, " +
//// "a.wechatId, " +
//// "a.mobile, " +
//// "a.createdTime, " +
//// "r.roleType " +
//// "FROM " +
//// "User a , " +
//// "User b , " +
//// "UserGroup m , " +
//// "UserGroup r " +
//// "WHERE " +
//// //"a.companyId = b.companyId " +
//// "a.companyId = :cid " +
//// "AND a.parentId = b.id " +
//// "AND a.userGroupId = r.id " +
//// "AND b.userGroupId = m.id " +
//// "AND a.isShadow <> :isShadow " +
//// "AND r.roleType not in :ugrt " +
//// "AND (a.userName like :name or a.account like :name) " +
//// "AND (a.del = :isDel ) " +
//// "order by a.createdTime desc")
//// Page<Object> findClientUserListFilterByRole(@Param("cid") Long companyId,
//// @Param("name") String filterName,
//// Pageable pageable,
//// @Param("isDel") Boolean isDel,
//// @Param("ugrt") List<Integer> userGroupRoleType,
//// @Param("isShadow") Integer isShadow);
//
//// @Query(value = "SELECT id,account,user_group_id FROM cl_user WHERE company_id = ?1 and del = ?2 ORDER BY created_time DESC ", nativeQuery = true)
//// List<Object[]> findAllByCompanyId(Long companyId, boolean del);
////
//// @Query(value = "SELECT * FROM cl_user WHERE id = ?1 AND status = ?2 AND del = ?3", nativeQuery = true)
//// User findByIdAndStatusAndDel(Long userId, Integer userNormal, Boolean userNotDelete);
////
//// @Query(value = "SELECT * FROM cl_user WHERE token = ?1 ", nativeQuery = true)
//// User findBytoken(String creator);
//
//// @Query("SELECT " +
//// "a.id ," +
//// "a.account , " +
//// "a.userName, " +
//// "a.email, " +
//// "a.wechat, " +
//// "a.wechatId, " +
//// "a.mobile, " +
//// "r.id AS userGroupId , " +
//// "r.name as userGroupName, " +
//// "a.status, " +
//// "a.createdTime, " +
//// "a.password " +
//// "FROM " +
//// "User a , " +
//// "UserGroup r " +
//// "WHERE a.token = :token " +
//// "AND a.userGroupId = r.id " +
//// "AND (a.del = :isDel ) " +
//// "order by a.createdTime desc")
//// List<Object[]> fetchUserByTokenAndDel(@Param("token") String token, @Param("isDel") Boolean userNotDelete);
//
//// @Query(value = "select g.channel_ids from cl_user u, cl_user_group g where u.token = ?1 AND g.id = u.user_group_id ", nativeQuery = true)
//// String getChannelIdsByToken(String token);
//}

1
cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/topic/ParseExcelTaskRepository.java

@ -1,7 +1,6 @@
package com.bfd.mf.common.web.repository.mysql.topic;
import com.bfd.mf.common.web.entity.mysql.topic.ParseExcelTask;
import com.bfd.mf.common.web.entity.mysql.topic.Subject;
import org.apache.ibatis.annotations.Options;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;

2
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/QueryRequest.java

@ -19,6 +19,7 @@ package com.bfd.mf.common.web.vo.params;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
@ -27,7 +28,6 @@ import java.util.List;
public class QueryRequest implements Serializable {
private static final long serialVersionUID = 1L;
// 必传字段
// private Long subjectId;
@ApiModelProperty(value = "subjectId" , required = true , notes = "专题ID")

88
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataLineCount.java

@ -1,88 +0,0 @@
///*
// * Copyright (C) 2016 Baifendian Corporation
// * <p>
// * Licensed under the Apache License, Version 2.0 (the "License");
// * you may not use this file except in compliance with the License.
// * You may obtain a copy of the License at
// * <p>
// * http://www.apache.org/licenses/LICENSE-2.0
// * <p>
// * Unless required by applicable law or agreed to in writing, software
// * distributed under the License is distributed on an "AS IS" BASIS,
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// * See the License for the specific language governing permissions and
// * limitations under the License.
// */
//
//package com.bfd.mf.common.web.vo.view.analysis;
//
///**
// * 折线图统计
// *
// * @author quanyou.chang
// * @create 2016-09-22 10:43
// **/
//public class DataLineCount extends DataCountBase implements Comparable<DataLineCount> {
// /**
// * 时间戳
// */
// private Long timestamp;
// /**
// *
// */
// private Long data;
//
// public Long getTimestamp() {
// return timestamp;
// }
//
// public void setTimestamp(Long timestamp) {
// this.timestamp = timestamp;
// }
//
//
// public String getName() {
// return name;
// }
//
// public void setName(String name) {
// this.name = name;
// }
//
// /**
// * 坐标
// */
// private String name;
//
//
// public Long getData() {
// return data;
// }
//
// public void setData(Long data) {
// this.data = data;
// }
//
// @Override
// public int compareTo(DataLineCount o) {
// return this.getTimestamp().compareTo(o.getTimestamp());
// }
// @Override
// public boolean equals(Object o) {
// if (this == o) return true;
// if (o == null || getClass() != o.getClass()) return false;
//
// DataLineCount that = (DataLineCount) o;
//
// if (timestamp != null ? !timestamp.equals(that.timestamp) : that.timestamp != null) return false;
// return name != null ? name.equals(that.name) : that.name == null;
//
// }
//
// @Override
// public int hashCode() {
// int result = timestamp != null ? timestamp.hashCode() : 0;
// result = 31 * result + (name != null ? name.hashCode() : 0);
// return result;
// }
//}

145
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorBaseEntity.java

@ -17,14 +17,12 @@
package com.bfd.mf.common.web.vo.view.monitor;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.constants.ESConstant;
import com.google.common.base.Objects;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -37,6 +35,8 @@ public class ESMonitorBaseEntity implements Comparable<ESMonitorBaseEntity>, Ser
private String author; // 作者
private String authorId; // 作者ID
private String avatar; // 作者头像
private String authornickname; // 作者昵称
private String userType;
private String title; // 文章title
private Long pubTime = 0L; //文章发布时间
@ -65,14 +65,29 @@ public class ESMonitorBaseEntity implements Comparable<ESMonitorBaseEntity>, Ser
private String commentId;
private String content; // 内容
private String contentSimHash; //文章SimHash
private Integer contentSize; //正文长度
// 数字
private String forwardContent;
/**
* 文章SimHash
*/
private String contentSimHash;
/**
* //正文长度
*/
private Integer contentSize;
/**
* ,收藏数阅读数好看数的 数字
*/
private String quoteCount ;
private String attitudesCount;
private Integer commentsCount = 0;
private String collectCount;
// 词云
private int readCount;
private int viewCnt;
/**
* 词云
*/
private List<String> hlKeyWords;
private List<String> places;
private List<String> hashTag;
@ -122,42 +137,101 @@ public class ESMonitorBaseEntity implements Comparable<ESMonitorBaseEntity>, Ser
private String promotionInfo;
private int readCount;
public int getReadCount() {
return readCount;
private String forumScore;
private String userTypeContent;
private String pageType;
private int hasImage;
private int hasVideo;
private int hasFile;
public int getViewCnt() {
return viewCnt;
}
public void setReadCount(int readCount) {
this.readCount = readCount;
public void setViewCnt(int viewCnt) {
this.viewCnt = viewCnt;
}
public int getHasImage() {
return hasImage;
}
public void setHasImage(int hasImage) {
this.hasImage = hasImage;
}
public int getHasVideo() {
return hasVideo;
}
public void setHasVideo(int hasVideo) {
this.hasVideo = hasVideo;
}
public int getHasFile() {
return hasFile;
}
public void setHasFile(int hasFile) {
this.hasFile = hasFile;
}
private int ugc;
private int egc;
private int pgc;
public String getPageType() {
return pageType;
}
public void setPageType(String pageType) {
this.pageType = pageType;
}
public String getUserTypeContent() {
return userTypeContent;
}
public void setUserTypeContent(String userTypeContent) {
this.userTypeContent = userTypeContent;
}
public String getUserType() {
return userType;
}
public void setUserType(String userType) {
this.userType = userType;
}
public int getUgc() {
return ugc;
public String getForumScore() {
return forumScore;
}
public void setUgc(int ugc) {
this.ugc = ugc;
public void setForumScore(String forumScore) {
this.forumScore = forumScore;
}
public int getEgc() {
return egc;
public String getForwardContent() {
return forwardContent;
}
public void setEgc(int egc) {
this.egc = egc;
public void setForwardContent(String forwardContent) {
this.forwardContent = forwardContent;
}
public int getPgc() {
return pgc;
public int getReadCount() {
return readCount;
}
public void setReadCount(int readCount) {
this.readCount = readCount;
}
public void setPgc(int pgc) {
this.pgc = pgc;
public String getAuthornickname() {
return authornickname;
}
public void setAuthornickname(String authornickname) {
this.authornickname = authornickname;
}
public String getPromotionInfo() {
@ -687,15 +761,16 @@ public class ESMonitorBaseEntity implements Comparable<ESMonitorBaseEntity>, Ser
}
public String getSysSentimentTag() {
if(sysSentiment < 0.5){
sysSentimentTag = "负面";
}
if(sysSentiment == 0.5){
sysSentimentTag = "中性";
}
if(sysSentiment > 0.5){
sysSentimentTag = "正面";
}
sysSentimentTag = "中性";
// if(sysSentiment < 0.5){
// sysSentimentTag = "负面";
// }
// if(sysSentiment == 0.5){
// sysSentimentTag = "中性";
// }
// if(sysSentiment > 0.5){
// sysSentimentTag = "正面";
// }
return sysSentimentTag;
}

2
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/monitor/ESMonitorEntity.java

@ -20,8 +20,6 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
public class ESMonitorEntity extends ESMonitorBaseEntity implements Serializable {

9
cl_search_api/src/main/java/com/bfd/mf/controller/SearchAnalysisController.java

@ -1,10 +1,10 @@
package com.bfd.mf.controller;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.service.SearchAnalysisService;
import com.bfd.mf.common.util.enums.RTCodeEnum;
import com.bfd.mf.common.web.component.wrapper.ResponseWrapper;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.service.SearchAnalysisService;
import com.bfd.nlp.common.util.constants.MediaTypes;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@ -12,7 +12,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
@Controller

3
cl_search_api/src/main/java/com/bfd/mf/controller/SearchAuthorController.java

@ -38,7 +38,8 @@ public class SearchAuthorController {
try {
JSONObject result;
String scorllId = queryRequest.getScrollId();
if(null != scorllId){ // 这个是导出要用的
// 这个是导出要用的
if(null != scorllId){
result = searchDataService.exportDataInSubjectIndex(queryRequest);
return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result);
}else{ // 这个是查询

127
cl_search_api/src/main/java/com/bfd/mf/controller/SearchDataController.java

@ -2,24 +2,31 @@ package com.bfd.mf.controller;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.service.SearchDataService;
import com.bfd.mf.common.util.enums.RTCodeEnum;
import com.bfd.mf.common.web.component.wrapper.ResponseWrapper;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.service.SearchDataService;
import com.bfd.mf.service.UpdateService;
import com.bfd.nlp.common.util.constants.MediaTypes;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;
import io.swagger.annotations.ApiOperation;
import com.bfd.nlp.common.util.constants.MediaTypes;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import java.util.ArrayList;
/**
* @author dujing
*/
@Controller
@RequestMapping("/crawl")
@Api(value="数据查询的控制器")
@ -56,9 +63,11 @@ public class SearchDataController {
}
return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result);
}else {
if (subjectId.equals("all") || subjectId.contains(",")) {// 全局数据查询
// 全局数据查询
if (subjectId.equals("all") || subjectId.contains(",")) {
result = searchDataService.queryDataList(queryRequest);
} else if (subjectId.equals("")) { // 没有专题
// 没有专题
} else if (subjectId.equals("")) {
result.put(ESConstant.ALLDOCNUMBER, 0L);
result.put(ESConstant.MONITORLISTS, new ArrayList<>());
} else {
@ -68,11 +77,11 @@ public class SearchDataController {
}
Integer allDocNumber = result.getIntValue(ESConstant.ALLDOCNUMBER);
Integer limit = queryRequest.getLimit();
Integer page = 1;
if(allDocNumber%limit==0){
int page = 1;
if(allDocNumber%limit == 0){
page = allDocNumber/limit;
}else{
page = allDocNumber/limit +1;
page = allDocNumber/limit + 1;
}
if(null != queryRequest.getPage() && !queryRequest.getPage().equals("")) {
if (page > 0 && queryRequest.getPage() > page) {
@ -83,7 +92,7 @@ public class SearchDataController {
logger.info("接口查询时长:statr:"+ start +" ; end:"+end + " ; time = " + (end - start) + " ; count = "+result.get(ESConstant.ALLDOCNUMBER));
return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result);
} catch (Exception e) {
logger.error("[queryData] Failed,The error message is :{}", e);
logger.error("[SearchDataController] queryDataList Failed,The error message is :{}", e);
return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed");
}
@ -98,8 +107,7 @@ public class SearchDataController {
@ApiImplicitParams({
@ApiImplicitParam(paramType="query", name = "subjectId", value = "专题ID", required = true, dataType = "String"),
@ApiImplicitParam(paramType="query", name = "docId", value = "主贴唯一ID", required = true, dataType = "String"),
@ApiImplicitParam(paramType="query", name = "siteId", value = "站点ID", required = true, dataType = "String"),
})
@ApiImplicitParam(paramType="query", name = "siteId", value = "站点ID", required = true, dataType = "String"),})
public JSONObject getInfo(String subjectId,String docId,String siteId) {
QueryRequest queryRequest = new QueryRequest();
queryRequest.setSubjectId(subjectId);
@ -154,14 +162,23 @@ public class SearchDataController {
}
/**
* 查询 数据的Counts 用户左侧的显示
* @param queryRequest
* @return
*/
@ApiOperation(value = "查询数据列表")
@RequestMapping(value = "/subject/queryCounts", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8)
@ResponseBody
public JSONObject queryDataCounts(@RequestBody QueryRequest queryRequest) {
logger.info("[queryDataCounts] partial / Params: {}", JSONObject.toJSONString(queryRequest));
try {
JSONObject result = searchDataService.queryDataCountsInOneIndex(queryRequest);
JSONObject result = new JSONObject();
if(null != queryRequest.getSubjectId()) {
result = searchDataService.queryDataCountsInOneIndex(queryRequest);
}else{
return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed");
}
return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result);
} catch (Exception e) {
logger.error("[queryDataCounts] Failed,The error message is :{}", e);
@ -192,20 +209,26 @@ public class SearchDataController {
}
/**
* 示例文件夹的专题复制
*/
@ApiOperation(value = "复制专题")
@RequestMapping(value = "/reindex/reindexSubject", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8)
@ResponseBody
public JSONObject reIndex(@RequestBody QueryRequest queryRequest){
logger.info("[reIndex] partial / Params: {}", JSONObject.toJSONString(queryRequest));
try {
JSONObject result = searchDataService.reIndexSubject(queryRequest);
return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result);
searchDataService.reIndexSubject(queryRequest);
} catch (Exception e) {
logger.error("[reIndex] Failed,The error message is :{}", e);
return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed");
}
return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, new JSONObject());
}
/**
* 删除专题的接口
*/
@ApiOperation(value = "删除专题")
@RequestMapping(value = "/delete/deleteBySubjectId", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8)
@ResponseBody
@ -220,4 +243,74 @@ public class SearchDataController {
}
}
@ApiOperation(value = "根据 cid 删除指定专题下的数据")
@RequestMapping(value = "/delete/deleteByCid", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8)
@ResponseBody
public JSONObject deleteSubjectByCid(@RequestBody QueryRequest queryRequest){
logger.info("[deleteSubject] partial / Params: {}", JSONObject.toJSONString(queryRequest));
try {
JSONObject result = searchDataService.deleteBySubjectIdByCid(queryRequest);
return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result);
} catch (Exception e) {
logger.error("[deleteSubject] Failed,The error message is :{}", e);
return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed");
}
}
@ApiOperation(value = "根据 crawlDataFlag 删除指定专题下的问题")
@RequestMapping(value = "/delete/deleteByCrawlDataFlag", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8)
@ResponseBody
public JSONObject deleteSubjectByCrawlDataFlag(@RequestBody QueryRequest queryRequest){
logger.info("[deleteSubject] partial / Params: {}", JSONObject.toJSONString(queryRequest));
try {
JSONObject result = searchDataService.deleteBySubjectIdByCrawlDataFlag(queryRequest);
return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result);
} catch (Exception e) {
logger.error("[deleteSubject] Failed,The error message is :{}", e);
return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed");
}
}
/**
* 获取json 结构数据
* @param queryRequest
* @return
*/
@ApiOperation(value = "查询数据列表")
@RequestMapping(value = "/subject/getJson", method = RequestMethod.POST, consumes = MediaTypes.JSON_UTF_8, produces = MediaTypes.JSON_UTF_8)
@ResponseBody
public JSONObject queryDataJsonList(@RequestBody QueryRequest queryRequest) {
logger.info("[queryDataList] partial / Params: {}", JSONObject.toJSONString(queryRequest));
try {
JSONObject result = new JSONObject();
long start = System.currentTimeMillis();
String scorllId = queryRequest.getScrollId();
// 专题数据导出
result = searchDataService.exportJsonDataInSubject(queryRequest);
Integer allDocNumber = result.getIntValue(ESConstant.ALLDOCNUMBER);
Integer limit = queryRequest.getLimit();
Integer page = 1;
if(allDocNumber%limit==0){
page = allDocNumber/limit;
}else{
page = allDocNumber/limit +1;
}
if(null != queryRequest.getPage() && !queryRequest.getPage().equals("")) {
if (page > 0 && queryRequest.getPage() > page) {
return ResponseWrapper.buildResponse(RTCodeEnum.C_SUBJECT_GRAMMAR_ERROR, "总数和分页不匹配");
}
}
long end = System.currentTimeMillis();
logger.info("接口查询时长:statr:"+ start +" ; end:"+end + " ; time = " + (end - start) + " ; count = "+result.get(ESConstant.ALLDOCNUMBER));
return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result);
} catch (Exception e) {
logger.error("[queryData] Failed,The error message is :{}", e);
return ResponseWrapper.buildResponse(RTCodeEnum.C_SERVICE_NOT_AVAILABLE, "Query failed");
}
}
}

8
cl_search_api/src/main/java/com/bfd/mf/controller/UploadExcelController.java

@ -12,7 +12,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
@ -37,8 +40,7 @@ public class UploadExcelController {
@ApiOperation(value = "上传Excel", httpMethod = "POST")
@PostMapping(value = "/SubjectsExcel")
@ResponseBody
public JSONObject insertExcelTask(@RequestParam("file") MultipartFile file,
@RequestParam("userId") String userId) {
public JSONObject insertExcelTask(@RequestParam("file") MultipartFile file, @RequestParam("userId") String userId) {
logger.info("[insertExcelTask] partial / Params: {}", userId);
try {
JSONObject jsonObject = new JSONObject();

9
cl_search_api/src/main/java/com/bfd/mf/service/SearchAnalysisService.java

@ -95,9 +95,12 @@ public class SearchAnalysisService {
Map<String,String> valueLabelMap = new HashMap<>();
Map<String,String> categoryLabelMap = new HashMap<>();
for (ESMonitorEntity esMonitorEntity : esMonitorEntityList) {
String docType = esMonitorEntity.getDocType(); // 页面类型
String valueLabel = esMonitorEntity.getValueLabel(); // 价值标签
String categoryLabel = esMonitorEntity.getCategoryLabel();// 分类标签
// 页面类型
String docType = esMonitorEntity.getDocType();
// 价值标签
String valueLabel = esMonitorEntity.getValueLabel();
// 分类标签
String categoryLabel = esMonitorEntity.getCategoryLabel();
channelMap.put(docType,getDocTypeNum(docType));
valueLabelMap.put(valueLabel,valueLabel);
categoryLabelMap.put(categoryLabel,categoryLabel);

5
cl_search_api/src/main/java/com/bfd/mf/service/SearchAuthorService.java

@ -19,7 +19,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.Serializable;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class SearchAuthorService extends CrudService<SentimentModify, SentimentRepository> implements Serializable {

271
cl_search_api/src/main/java/com/bfd/mf/service/SearchDataService.java

@ -24,8 +24,8 @@ import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import springfox.documentation.spring.web.json.Json;
import java.io.Serializable;
import java.util.*;
@ -129,7 +129,8 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
String source = sourceAsMap.get(ESConstant.SOURCE).toString();
String docType = sourceAsMap.get(ESConstant.DOC_TYPE).toString();
String channel = sourceAsMap.get(ESConstant.CHANNEL).toString();
if (docType.equals(ESConstant.SOCIAL)) {
String enSource = sourceAsMap.get(ESConstant.EN_SOURCE).toString();
if (docType.equals(ESConstant.SOCIAL) && !enSource.equals("weixin") ) {
title = sourceAsMap.get(ESConstant.AUTHOR).toString();
}
@ -151,6 +152,15 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
is = false;
}
String vodeoUrl = sourceAsMap.get(ESConstant.VIDEOURL).toString();
Double sentiment = 0.5;
if(sourceAsMap.containsKey(ESConstant.SYS_SENTIMENT) && !sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString().contains("0.0")) {
sentiment = Double.valueOf(sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString());
}
String forumScore = "";
if(sourceAsMap.containsKey("forumScore")){
forumScore = (String) sourceAsMap.get("forumScore");
}
ESMonitorEntity esMonitorEntity = new ESMonitorEntity();
try {
@ -168,7 +178,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
esMonitorEntity.setUrl(url);
esMonitorEntity.setVideoUrl(vodeoUrl);
esMonitorEntity.setPostSource((String) sourceAsMap.getOrDefault(ESConstant.POST_SOURCE, ""));
esMonitorEntity.setSysSentiment(Double.valueOf(sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString()));
esMonitorEntity.setSysSentiment(sentiment);
esMonitorEntity.setCommentsCount(Integer.valueOf(sourceAsMap.getOrDefault(ESConstant.COMMENTS_COUNT, 0).toString()));
esMonitorEntity.setQuoteCount(sourceAsMap.getOrDefault(ESConstant.QUOTE_COUNT, 0).toString());
@ -181,6 +191,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
esMonitorEntity.setVideoPath(videoPath);
// 返回索引名称查询详情时使用
esMonitorEntity.setSubjectId(indexName);
esMonitorEntity.setForumScore(forumScore);
} catch (Exception e) {
logger.info("[SearchDataService] parseMainMessage ERROR !");
}
@ -199,6 +210,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
String channel = sourceAsMap.get(ESConstant.CHANNEL).toString();
String title = "标题为空";
String content = "";
String enSource = sourceAsMap.get(ESConstant.EN_SOURCE).toString();
if (sourceAsMap.containsKey(ESConstant.TITLE)) {
title = sourceAsMap.get(ESConstant.TITLE).toString();
@ -206,27 +218,30 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
if(searchType == 2){
title = "";
}else {
if (docType.equals(ESConstant.SOCIAL)) {
if (docType.equals(ESConstant.SOCIAL) && !enSource.equals("weixin")) {
title = sourceAsMap.get(ESConstant.AUTHOR).toString();
}
}
String author = "";
String authornickname = "";
String quoteCount = "0";
String attitudeCount = "0";
String enSource = sourceAsMap.get(ESConstant.EN_SOURCE).toString();
// String enSource = sourceAsMap.get(ESConstant.EN_SOURCE).toString();
String source = "";
String price = "";
String collentCount = "0";
Integer viewCnt = 0;
if (sourceAsMap.containsKey(ESConstant.SOURCE)) {
source = sourceAsMap.get(ESConstant.SOURCE).toString();
}
String siteId = "";
String icon = "";
String siteType = "";
if(enSource.equals(ESConstant.SINA)){
siteId = "183"; // 微博的我忘了为啥这个需要特殊处理
}else if(siteMap.containsKey(enSource)){
// if(enSource.equals(ESConstant.SINA)){
// siteId = "183"; // 微博的我忘了为啥这个需要特殊处理
// }else
if(siteMap.containsKey(enSource)){
Map<String, Object> siteOtherMap = siteMap.get(enSource);
if (siteOtherMap.containsKey("site_id")) {
siteId = siteMap.get(enSource).get("site_id").toString();
@ -243,7 +258,11 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
if(sourceAsMap.containsKey(ESConstant.PRODUCTPARAMETER)){
productParameter = sourceAsMap.get(ESConstant.PRODUCTPARAMETER).toString();
}
if (searchType == 0 && docType.equals(ESConstant.ITEM)) { // 电商主贴
if(sourceAsMap.containsKey(ESConstant.AUTHORNICKNAME)){
authornickname = sourceAsMap.get(ESConstant.AUTHORNICKNAME).toString();
}
// 电商主贴
if (searchType == 0 && docType.equals(ESConstant.ITEM)) {
content = productParameter;
if (!sourceAsMap.get(ESConstant.AUTHORNICKNAME).equals("")) {
author = sourceAsMap.get(ESConstant.AUTHORNICKNAME).toString();
@ -259,13 +278,16 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
price = sourceAsMap.get(ESConstant.PRICE).toString();
attitudeCount = sourceAsMap.get(ESConstant.POSTCOUNT).toString();
} else if(searchType == 2){
// 用户
author = sourceAsMap.get(ESConstant.AUTHOR).toString();
content = productParameter;
}else {
// 其他主贴
content = sourceAsMap.get(ESConstant.CONTENT).toString();
author = sourceAsMap.get(ESConstant.AUTHOR).toString();
quoteCount = sourceAsMap.get(ESConstant.QUOTE_COUNT).toString();
collentCount = sourceAsMap.get(ESConstant.COLLE_CTCOUNT).toString();
viewCnt = Integer.valueOf(sourceAsMap.get(ESConstant.VIEW_CNT).toString());
if(sourceAsMap.get(ESConstant.ATTITUDES_COUNT).toString().contains(ESConstant.TOTALCOUNT)) {
JSONObject countMap = JSONObject.parseObject(sourceAsMap.get(ESConstant.ATTITUDES_COUNT).toString());
attitudeCount = countMap.getString(ESConstant.TOTALCOUNT);
@ -322,13 +344,15 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
List filePathSize = new ArrayList();
if(sourceAsMap.containsKey(ESConstant.FILEPATHSIZE)) {
if (!("").equals(sourceAsMap.get(ESConstant.FILEPATHSIZE)) && null != sourceAsMap.get(ESConstant.FILEPATHSIZE)) {
filePathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.FILEPATHSIZE).toString());
// filePathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.FILEPATHSIZE).toString());
filePathSize = (List) sourceAsMap.get(ESConstant.FILEPATHSIZE);
}
}
List imagePathSize = new ArrayList();
if(sourceAsMap.containsKey(ESConstant.IMAGEPATHSIZE)) {
if (null != sourceAsMap.get(ESConstant.IMAGEPATHSIZE) && !("[]").equals(sourceAsMap.get(ESConstant.IMAGEPATHSIZE))) {
imagePathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.IMAGEPATHSIZE).toString());
// imagePathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.IMAGEPATHSIZE).toString());
imagePathSize = (List) sourceAsMap.get(ESConstant.IMAGEPATHSIZE);
}
}
List videoPathSize = new ArrayList();
@ -337,7 +361,13 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
&& !("[]").equals(sourceAsMap.get(ESConstant.VIDEOPATHSIZE))
&& !("{\"\":null}").equals(sourceAsMap.get(ESConstant.VIDEOPATHSIZE).toString())) {
if(sourceAsMap.get(ESConstant.VIDEOPATHSIZE).toString().contains(ESConstant.URL)) {
videoPathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.VIDEOPATHSIZE).toString());
if(sourceAsMap.get(ESConstant.VIDEOPATHSIZE) instanceof String) {
videoPathSize = JSONObject.parseArray(sourceAsMap.get(ESConstant.VIDEOPATHSIZE).toString());
}else{
videoPathSize = (List) sourceAsMap.get(ESConstant.VIDEOPATHSIZE);
}
//java.lang.String cannot be cast to java.util.List
//
}
}
}
@ -355,15 +385,9 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
String translateContent = "";
if (sourceAsMap.containsKey(ESConstant.TRANSLATETITLE)) {
translateTitle = (String) sourceAsMap.get(ESConstant.TRANSLATETITLE);
if(translateTitle.equals("")){
translateTitle = "未获取到翻译标题";
}
}
if (sourceAsMap.containsKey(ESConstant.TRANSLATECONTENT)) {
translateContent = (String) sourceAsMap.get(ESConstant.TRANSLATECONTENT);
if(translateContent.equals("")){
translateContent = "未获取到翻译正文";
}
}
// 词云
List<String> hlKeywords = new ArrayList<>();
@ -435,28 +459,51 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
if(sourceAsMap.containsKey(ESConstant.PROMOTIONINFO)){
promotionInfo = sourceAsMap.get(ESConstant.PROMOTIONINFO).toString();
}
int ugc = 0;
int egc = 0;
int pgc = 0;
if(sourceAsMap.containsKey(ESConstant.SEARCH_SCOPE_UGC)){
ugc = (int) sourceAsMap.get(ESConstant.SEARCH_SCOPE_UGC);
int hasFile = 0;
int hasVideo = 0;
int hasImage = 0;
if(sourceAsMap.containsKey(ESConstant.HAS_FILE)){
hasFile = (int) sourceAsMap.get(ESConstant.HAS_FILE);
}
if(sourceAsMap.containsKey(ESConstant.SEARCH_SCOPE_EGC)){
egc = (int) sourceAsMap.get(ESConstant.SEARCH_SCOPE_EGC);
if(sourceAsMap.containsKey(ESConstant.HAS_VIDEO)){
hasVideo = (int) sourceAsMap.get(ESConstant.HAS_VIDEO);
}
if(sourceAsMap.containsKey(ESConstant.SEARCH_SCOPE_PGC)){
pgc = (int) sourceAsMap.get(ESConstant.SEARCH_SCOPE_PGC);
if(sourceAsMap.containsKey(ESConstant.HAS_IMAGE)){
hasImage = (int) sourceAsMap.get(ESConstant.HAS_IMAGE);
}
int readCount = 0;
if(sourceAsMap.containsKey("readCount") && sourceAsMap.get("readCount") != ""){
readCount = (int) sourceAsMap.get("readCount");
}
Double sentiment = 0.5;
if(sourceAsMap.containsKey(ESConstant.SYS_SENTIMENT)
&& !sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString().contains("0.0")) {
sentiment = Double.valueOf(sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString());
}
String forumScore = "";
if(sourceAsMap.containsKey("forumScore")){
forumScore = (String) sourceAsMap.get("forumScore");
}
String userType = "";
if(sourceAsMap.containsKey("userType")){
userType = (String) sourceAsMap.get("userType");
}
String userTypeContent = "";
if(sourceAsMap.containsKey("userTypeContent")){
userTypeContent = (String) sourceAsMap.get("userTypeContent");
}
String pageType = "";
if(sourceAsMap.containsKey("pageType")){
pageType = sourceAsMap.get("pageType").toString();
}
try {
esMonitorEntity.setForwardContent(sourceAsMap.get("forwardContent").toString());
esMonitorEntity.setReadCount(readCount);
esMonitorEntity.setEgc(egc);
esMonitorEntity.setPgc(pgc);
esMonitorEntity.setUgc(ugc);
esMonitorEntity.setHasFile(hasFile);
esMonitorEntity.setHasVideo(hasVideo);
esMonitorEntity.setHasImage(hasImage);
esMonitorEntity.setDataId(dataId);
esMonitorEntity.setDocId(docId);
esMonitorEntity.setChannel(channel);
@ -480,6 +527,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
esMonitorEntity.setQuoteCount(quoteCount);
esMonitorEntity.setAttitudesCount(attitudeCount);
esMonitorEntity.setCollectCount(collentCount);
esMonitorEntity.setViewCnt(viewCnt);
// 视频音频
esMonitorEntity.setOcrText(ocrText);
esMonitorEntity.setAsrText(asrText);
@ -497,7 +545,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
esMonitorEntity.setIcon(icon);
esMonitorEntity.setVideoUrl(vodeoUrl);
esMonitorEntity.setPostSource((String) sourceAsMap.getOrDefault(ESConstant.POST_SOURCE, ""));
esMonitorEntity.setSysSentiment(Double.valueOf(sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString()));
esMonitorEntity.setSysSentiment(sentiment);
esMonitorEntity.setIsDownload(is);
esMonitorEntity.setFilePath(filePath);
esMonitorEntity.setImagePath(imagePath);
@ -519,6 +567,15 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
// 二次评论的字段
esMonitorEntity.setOtherSourceJson(otherSourceJson);
esMonitorEntity.setPromotionInfo(promotionInfo);
// 用户昵称
esMonitorEntity.setAuthornickname(authornickname);
esMonitorEntity.setForumScore(forumScore);
// 用户Type
esMonitorEntity.setUserType(userType);
esMonitorEntity.setUserTypeContent(userTypeContent);
esMonitorEntity.setPageType(pageType);
} catch (Exception e) {
e.printStackTrace();
@ -535,7 +592,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
List<String> currentIndexList = new ArrayList<>();
String subjectId = queryRequest.getSubjectId();
if(null != queryRequest.getSubjectId() && !("").equals(subjectId)){ // 如果是专题去专题的索引查就行
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId;
currentIndexList.add(subjectId);
}else{ // 如果是全部数据就直接去 渠道对应的索引查渠道可以从 docId 中截取出来
@ -628,10 +685,10 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
// }else {
boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.PRIMARY, 0));
// }
SearchRequestBuilder builder = esServerUtils
.buildSearchRequestBuilder(cluster.getId(), currentIndexList)
SearchRequestBuilder builder = esServerUtils.buildSearchRequestBuilder(cluster.getId(), currentIndexList)
.setQuery(boolQueryBuilder)
.setFetchSource(ESConstant.COMMENT_FIELD_DATA, null);
SearchResponse searchResponse = builder.execute().actionGet();
SearchHit[] response = searchResponse.getHits().getHits();
@ -658,7 +715,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
List<String> currentIndexList = new ArrayList<>();
String subjectId = queryRequest.getSubjectId();
if(!("").equals(subjectId) && null != subjectId){ // 如果是专题去专题的索引查就行
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId;
}else{ // 如果是全部数据就直接去 渠道对应的索引查渠道可以从 docId 中截取出来
logger.info("[SearchDataService] queryOneDataByDocId 查询 全局数据 : {}" , subjectId);
@ -670,7 +727,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
JSONObject jsonObject = getOneDataByDocId(docId, cluster, currentIndexList);
jsonObject = setLable(jsonObject);
// 替换几个 pathSize 中的链接的前缀
if(jsonObject.containsKey(ESConstant.IMAGEPATHSIZE)){ //http://172.18.1.113:8080
if(jsonObject.containsKey(ESConstant.IMAGEPATHSIZE)){
List<Map<String,String>>imagePathSize = (List<Map<String, String>>) jsonObject.get(ESConstant.IMAGEPATHSIZE);
for (Map<String,String> imagePath: imagePathSize) {
String url = imagePath.get(ESConstant.URL);
@ -678,7 +735,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
imagePath.put(ESConstant.URL,url);
}
}
if(jsonObject.containsKey(ESConstant.VIDEOPATHSIZE)){ //http://172.18.1.113:8080
if(jsonObject.containsKey(ESConstant.VIDEOPATHSIZE)){
List<Map<String,String>>videoPathSize = (List<Map<String, String>>) jsonObject.get(ESConstant.VIDEOPATHSIZE);
for (Map<String,String> videoPath: videoPathSize) {
String url = videoPath.get(ESConstant.URL);
@ -705,9 +762,13 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
jsonObject = getUserMessageByAuthorId(jsonObject,docType,queryRequest,subjectId);
// 如果是电商数据需要把电商详情信息填到 content 并把一些店铺信息填到 author中
jsonObject = getItemDataShopMeggage(jsonObject,docType);
return jsonObject;
}
// 崔老师那个版本需要添加 分类标签和 价值标签
private JSONObject setLable(JSONObject jsonObject) {
if(!jsonObject.containsKey(ESConstant.VALUELABEL) || null == jsonObject.get(ESConstant.VALUELABEL)){
@ -759,24 +820,6 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
newJsonObject.put(ESConstant.SOURCE,jsonObject.get(ESConstant.SOURCE));
newJsonObject.put(ESConstant.URL,jsonObject.get(ESConstant.URL));
jsonObject.put(ESConstant.AUTHOR,newJsonObject);
if(jsonObject.containsKey(ESConstant.SMALLIMGS) && jsonObject.get(ESConstant.SMALLIMGS) instanceof List){
List<String> smallImgs = (List<String>) jsonObject.get(ESConstant.SMALLIMGS);
List<Map<String,String>> imagePathSize = new ArrayList<>();
if(null != smallImgs && smallImgs.size() > 0) { // 由于电商的图片不做下载因此输出到页面的时候写死了大小和分辨率
for (Object img : smallImgs) {
if (!img.toString().contains("http")) {
Map<String,String> imagePathMap = new HashMap<>();
String url = "http:" + img ;
imagePathMap.put(ESConstant.URL ,url);
imagePathMap.put(ESConstant.SIZE ,"4KB");
imagePathMap.put(ESConstant.VIDEOTIME ,"");
imagePathMap.put(ESConstant.RESOLUTION ,"50×50");
imagePathSize.add(imagePathMap);
}
}
}
jsonObject.put(ESConstant.IMAGEPATHSIZE,imagePathSize);
}
}
return jsonObject;
}
@ -798,12 +841,15 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
*/
private JSONObject socialDataChangeAuthorAndTitle(JSONObject jsonObject) {
if(jsonObject.containsKey(ESConstant.DOC_TYPE)){
// if(jsonObject.get(ESConstant.EN_SOURCE).equals(ESConstant.TWITTER)
// || jsonObject.get(ESConstant.EN_SOURCE).equals(ESConstant.FACEBOOK)
// || jsonObject.get(ESConstant.EN_SOURCE).equals(ESConstant.SINA)) {
if(jsonObject.get(ESConstant.DOC_TYPE).equals(ESConstant.SOCIAL)){
String author = jsonObject.getString(ESConstant.AUTHOR);
jsonObject.put(ESConstant.TITLE,author);
String enSource = jsonObject.getString(ESConstant.EN_SOURCE);
if(!enSource.equals("weixin")) {
jsonObject.put(ESConstant.TITLE, author);
// content 的值 放入到 译文Title 中是为了展示一下翻译这个后面还是删掉吧
String content = jsonObject.getString(ESConstant.CONTENT);
jsonObject.put(ESConstant.TRANSLATECONTENT,content);
}
if(jsonObject.get(ESConstant.EN_SOURCE).equals(ESConstant.SINA)){
jsonObject.put(ESConstant.SOURCE,"微博");
}
@ -826,31 +872,32 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
.setFetchSource(ESConstant.FIELD_DATA, null);
SearchResponse searchDataResponse = builder.execute().actionGet();
System.out.println("本次查询结果条数:"+searchDataResponse.getHits().totalHits);
if(searchDataResponse.getHits().getHits().length >0) {
Map<String, Object> result = searchDataResponse.getHits().getHits()[0].getSourceAsMap();
if (result.size() > 0) {
result.entrySet().stream()
.forEach(entry -> {
if (entry.getKey().equals(ESConstant.FILEPATH) && entry.getValue().equals("")) {
jsonObject.put(entry.getKey(), new ArrayList<>());
} else if (entry.getKey().equals(ESConstant.FILEPATHSIZE) ||
entry.getKey().equals(ESConstant.IMAGEPATHSIZE) ||
entry.getKey().equals(ESConstant.VIDEOPATHSIZE)) {
if(entry.getValue().toString() .contains(ESConstant.URL)) {
jsonObject.put(entry.getKey(), JSONObject.parseArray(entry.getValue().toString()));
}else{
jsonObject.put(entry.getKey(),new ArrayList<>());
}
} else if(entry.getKey().equals(ESConstant.ATTITUDES_COUNT)){
if(entry.getValue().toString().contains(ESConstant.TOTALCOUNT)){
JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString());
jsonObject.put(entry.getKey(),totalCount.get(ESConstant.TOTALCOUNT));
}
} else {
if (result.size() > 0) { result.entrySet().stream()
.forEach(entry -> {
if (entry.getKey().equals(ESConstant.FILEPATH) && entry.getValue().equals("")) {
jsonObject.put(entry.getKey(), new ArrayList<>());
} else if (entry.getKey().equals(ESConstant.FILEPATHSIZE)
|| entry.getKey().equals(ESConstant.IMAGEPATHSIZE)
|| entry.getKey().equals(ESConstant.VIDEOPATHSIZE)) {
if(entry.getValue().toString() .contains(ESConstant.URL)) {
jsonObject.put(entry.getKey(), entry.getValue());
}else{
jsonObject.put(entry.getKey(),new ArrayList<>());
}
});
} else if(entry.getKey().equals(ESConstant.ATTITUDES_COUNT)){
if(entry.getValue().toString().contains(ESConstant.TOTALCOUNT)){
JSONObject totalCount = JSONObject.parseObject(entry.getValue().toString());
jsonObject.put(entry.getKey(),totalCount.get(ESConstant.TOTALCOUNT));
}else{
jsonObject.put(entry.getKey(),entry.getValue());
}
} else {
jsonObject.put(entry.getKey(), entry.getValue());
}
});
}
}
return jsonObject;
@ -865,7 +912,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
List<String> currentIndexList = new ArrayList<>();
// 获取ES的参数及要查询的索引列表
String subjectId = queryRequest.getSubjectId();
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 special_cluster_type
currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster,subjectId);
Long clusterId = cluster.getId();
logger.info("[SearchDataService] queryDataList clusterId : {}; currentIndexList : {}" ,clusterId, currentIndexList.toString());
@ -947,7 +994,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
public JSONObject queryDataInOneIndex(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject();
try {
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
Long clusterId = cluster.getId();
logger.info("[SearchDataService] queryDataInOneIndex: clusterId :{} ; currentIndexList : {}", clusterId,currentIndexList.toString());
@ -977,8 +1024,8 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
public JSONObject exportDataInSubjectIndex(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject();
try {
// 获取 ES 的连接方式及要查询的索引列表
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 专题索引
// 获取 ES 的连接方式及要查询的索引列表 专题索引 special_cluster_type
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type);
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
Long clusterId = cluster.getId();
String [] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
@ -996,6 +1043,22 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
return jsonObject;
}
public JSONObject exportJsonDataInSubject(QueryRequest queryRequest){
JSONObject jsonObject = new JSONObject();
try{
// 获取 ES 的连接方式及要查询的索引列表
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 专题索引 special_cluster_type
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
Long clusterId = cluster.getId();
String [] indexName = currentIndexList.toArray(new String[currentIndexList.size()]);
logger.info("[SearchDataService] exportDataInSubjectIndex : IndexName :{} ; clusterId :{} ; currentIndexList :{}",indexName[0], clusterId , currentIndexList.toString());
jsonObject= esQueryServiceForSQMini.exportDataFromOneSubject(indexName, queryRequest);
}catch (Exception e){
e.printStackTrace();
}
return jsonObject;
}
public JSONObject exportDataInDateIndex(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject();
try {
@ -1007,7 +1070,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
currentIndexList = subjectQueryDataService.getIndexListByTimeRange(cluster, queryRequest.getStartTime(), queryRequest.getEndTime());
}else if(subjectId.contains(",")){
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111
currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
}
Long clusterId = cluster.getId();
@ -1028,7 +1091,7 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
public JSONObject queryDataCountsInOneIndex(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject();
try{
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
Cluster cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.special_cluster_type); // 111 special_cluster_type
List<String> currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
Long clusterId = cluster.getId();
logger.info("[SearchDataService] queryDataCountsInOneIndex: clusterId : {}; currentIndexList : {}" , clusterId ,currentIndexList.toString());
@ -1056,10 +1119,17 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
return jsonObject;
}
@Async
public JSONObject reIndexSubject(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject();
// 示例数据模板所在的索引
String oldIndex = "cl_major_12962";
String newIndexPre = "cl_special_1.0_";
String newIndex = newIndexPre + queryRequest.getSubjectId();
try {
// 将上面专题的数据复制到新的专题下
long created = esQueryServiceForSQMini.reIndexData(oldIndex,newIndex);
jsonObject.put("created",created);
}catch (Exception e){
e.printStackTrace();
}
@ -1068,11 +1138,38 @@ public class SearchDataService extends CrudService<SentimentModify, SentimentRep
public JSONObject deleteBySubjectId(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject();
String indexName = bfdApiConfig.getIndexNamePre()+queryRequest.getSubjectId();
try {
esQueryServiceForSQMini.deleteBySubjectId(indexName);
}catch (Exception e){
e.printStackTrace();
}
return jsonObject;
}
public JSONObject deleteBySubjectIdByCid(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject();
String indexName = bfdApiConfig.getIndexNamePre()+ queryRequest.getSubjectId();
String cid = queryRequest.getCid();
try {
esQueryServiceForSQMini.deleteBySubjectIdByCid(indexName,cid);
}catch (Exception e){
e.printStackTrace();
}
return jsonObject;
}
public JSONObject deleteBySubjectIdByCrawlDataFlag(QueryRequest queryRequest) {
JSONObject jsonObject = new JSONObject();
String indexName = bfdApiConfig.getIndexNamePre()+ queryRequest.getSubjectId();
String cid = queryRequest.getCid();
String crawlDataFlag = queryRequest.getCrawlDataFlag();
try {
esQueryServiceForSQMini.deleteBySubjectIdByCrawlDataFlag(indexName,cid,crawlDataFlag);
}catch (Exception e){
e.printStackTrace();
}
return jsonObject;
}
}

18
cl_search_api/src/main/java/com/bfd/mf/service/SearchKeywordsCouldService.java

@ -31,10 +31,14 @@ public class SearchKeywordsCouldService{
int topSize = 200;
try {
Map<String, Integer> keyWordsMaps = new HashMap<>();
Map<String, Integer> placesWordsMaps = new HashMap<>(); // 地点
Map<String, Integer> emojiWordsMaps = new HashMap<>(); // 表情
Map<String, Integer> hashTagWordsMaps = new HashMap<>(); // 话题
Map<String, Integer> opinionsWordsMaps = new HashMap<>(); // 评价
// 地点
Map<String, Integer> placesWordsMaps = new HashMap<>();
// 表情
Map<String, Integer> emojiWordsMaps = new HashMap<>();
// 话题
Map<String, Integer> hashTagWordsMaps = new HashMap<>();
// 评价
Map<String, Integer> opinionsWordsMaps = new HashMap<>();
// List<String> hlKeywordsList = new ArrayList<>();
for (ESMonitorEntity esMonitorEntity : esMonitorEntityList) {
List<String> hlKeyWords = esMonitorEntity.getHlKeyWords();
@ -63,10 +67,12 @@ public class SearchKeywordsCouldService{
CollectionUtils.sortByValueForListSubTopSize(placesWordsMaps, topSize * 2, placesWordsResultMap);
//sortByValueForListSubTopSize
Long keyWordsStart = System.currentTimeMillis();
jsonObject.put(ConditionCommon.WORD_CLOUD, CollectionUtils.sortByValueForList(textService.post(keyWordsResultMap, 1), topSize));
//jsonObject.put(ConditionCommon.WORD_CLOUD, CollectionUtils.sortByValueForList(textService.post(keyWordsResultMap, 1), topSize));
jsonObject.put(ConditionCommon.WORD_CLOUD,CollectionUtils.sortByValueForList(keyWordsResultMap, topSize));
logger.info("[keysWords Execute Time one] the time used is {} ms", System.currentTimeMillis() - keyWordsStart);
Long placesWordsStart = System.currentTimeMillis();
jsonObject.put(ConditionCommon.PLACE_CLOUD, CollectionUtils.sortByValueForList(textService.post(placesWordsResultMap, 2), topSize));
// jsonObject.put(ConditionCommon.PLACE_CLOUD, CollectionUtils.sortByValueForList(textService.post(placesWordsResultMap, 2), topSize));
jsonObject.put(ConditionCommon.PLACE_CLOUD, CollectionUtils.sortByValueForList(placesWordsResultMap, topSize));
logger.info("[placeWords Exceute Time two] the time used is {} ms", System.currentTimeMillis() - placesWordsStart);
jsonObject.put(ConditionCommon.HASH_TAG_CLOUD, CollectionUtils.sortByValueForList(hashTagWordsMaps, topSize));
jsonObject.put(ConditionCommon.EMOJI_CLOUD, CollectionUtils.sortByValueForList(emojiWordsMaps, topSize));

8
cl_search_api/src/main/java/com/bfd/mf/service/UpdateService.java

@ -40,7 +40,8 @@ public class UpdateService extends CrudService<SentimentModify, SentimentReposit
// 注册数据查询来源
clusterName = bfdApiConfig.esMiniName();
String sourceAddress [] = bfdApiConfig.esMiniAddress();
EsUtils.registerCluster(clusterName, sourceAddress);// 配置文件中的 es-source
// 配置文件中的 es-source
EsUtils.registerCluster(clusterName, sourceAddress);
}
@Override
@ -57,8 +58,9 @@ public class UpdateService extends CrudService<SentimentModify, SentimentReposit
Cluster cluster = null;
List<String> currentIndexList = new ArrayList<>();
if(!("").equals(subjectId) && null != subjectId){ // 如果是专题去专题的索引查就行
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
// 如果是专题去专题的索引查就行
if(!("").equals(subjectId) && null != subjectId){
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type);
subjectId = cluster.getPrefixIndexPattern() +"_"+ subjectId;
}
currentIndexList.add(subjectId);

11
cl_search_api/src/main/java/com/bfd/mf/service/UploadExcelService.java

@ -6,7 +6,6 @@ import com.bfd.mf.common.util.es.EsUtils2;
import com.bfd.mf.common.web.entity.mysql.topic.ParseExcelTask;
import com.bfd.mf.common.web.repository.mysql.topic.ParseExcelTaskRepository;
import com.bfd.mf.config.BFDApiConfig;
import org.assertj.core.util.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -14,10 +13,16 @@ import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.PostConstruct;
import java.io.*;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.math.BigInteger;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;

12
cl_stream_3.1.2.iml

@ -1,12 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

4
pom.xml

@ -5,8 +5,8 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.bfd.mf</groupId>
<artifactId>cl_stream_3.1.2</artifactId>
<version>3.1.2-SNAPSHOT</version>
<artifactId>cl_stream_3.2</artifactId>
<version>3.2-SNAPSHOT</version>
<packaging>pom</packaging>
<modules>

Loading…
Cancel
Save