zhicheng
/
swcl_search_api
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

20210607-v1 

崔老师项目版本代码
release-1.0
杜静 4 years ago
parent
dbf7d37942
commit
27b846360a
9 changed files with 54 additions and 40 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      cl_query_data_job/src/main/java/com/bfd/mf/job/service/backtrace/BacktraceService.java
  2. 4
      cl_query_data_job/src/main/java/com/bfd/mf/job/service/query/QueryService.java
  3. 25
      cl_query_data_job/src/main/java/com/bfd/mf/job/service/statistics/StatisticsService.java
  4. 33
      cl_query_data_job/src/main/resources/application.yml
  5. 4
      cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java
  6. 10
      cl_search_api/src/main/java/com/bfd/mf/controller/SearchDataController.java
  7. 4
      cl_search_api/src/main/resources/application.yml
  8. 4
      logs/sdkclient_error.log
  9. 0
      logs/sdkclient_error.log.2021-05-12-14

8
cl_query_data_job/src/main/java/com/bfd/mf/job/service/backtrace/BacktraceService.java
View File

@ -53,11 +53,11 @@ public class BacktraceService {
@PostConstruct
public void init() {
// 注册数据查询来源
EsUtils.registerCluster(config.esNormalClusterName(), config.esNormalAddress());// 配置文件中的 es-source
// EsUtils.registerCluster(config.esNormalClusterName(), config.esNormalAddress());// 配置文件中的 es-source
EsUtils.registerCluster(config.esMiniClusterName(), config.esMiniAddress()); // 配置文件中的 es-target
pRateLimiter = RateLimiter.create(1.0D / config.getPeriodS());
cRateLimiter = RateLimiter.create(1.0D / config.getPeriodS());
dataRateLimiter = RateLimiter.create(config.esMiniBulkRate());
// pRateLimiter = RateLimiter.create(1.0D / config.getPeriodS());
// cRateLimiter = RateLimiter.create(1.0D / config.getPeriodS());
// dataRateLimiter = RateLimiter.create(config.esMiniBulkRate());
// kafkaProducer = Kafka010Utils.getProducer(config.getBrokerList());
}

4
cl_query_data_job/src/main/java/com/bfd/mf/job/service/query/QueryService.java
View File

@ -61,9 +61,9 @@ public class QueryService {
@PostConstruct
public void init() {
// 注册数据查询来源
EsUtils.registerCluster(config.esNormalClusterName(), config.esNormalAddress());// 配置文件中的 es-source
// EsUtils.registerCluster(config.esNormalClusterName(), config.esNormalAddress());// 配置文件中的 es-source
EsUtils.registerCluster(config.esMiniClusterName(), config.esMiniAddress()); // 配置文件中的 es-target
pRateLimiter = RateLimiter.create(1.0D / config.getPeriodS());
// pRateLimiter = RateLimiter.create(1.0D / config.getPeriodS());
kafkaProducer = Kafka010Utils.getProducer(config.getBrokerList());
// cRateLimiter = RateLimiter.create(1.0D / config.getPeriodS());

25
cl_query_data_job/src/main/java/com/bfd/mf/job/service/statistics/StatisticsService.java
View File

@ -38,7 +38,7 @@ public class StatisticsService {
@PostConstruct
public void init() {
// 注册数据查询来源
EsUtils.registerCluster(config.esNormalClusterName(), config.esNormalAddress());// 配置文件中的 es-source
// EsUtils.registerCluster(config.esNormalClusterName(), config.esNormalAddress());// 配置文件中的 es-source
EsUtils.registerCluster(config.esMiniClusterName(), config.esMiniAddress()); // 配置文件中的 es-target
}
@ -49,14 +49,14 @@ public class StatisticsService {
LOGGER.info("------------------------------------------------------------------ StatisticsService ------------------------------------------------------");
long start = System.currentTimeMillis();
//-------统计134上的总量------------------------------------------------------------------------------------
String clusterName = config.esNormalClusterName(); // 获得 134 的 clusterName
statisticsTotal(clusterName);
// String clusterName = config.esNormalClusterName(); // 获得 134 的 clusterName
//statisticsTotal(clusterName);
long end = System.currentTimeMillis();
LOGGER.info("Statistics Total, took:{} ms.",(end - start));
//-------统计147上的 每个任务的总量-------------------------------------------------------------------------
start = System.currentTimeMillis();
clusterName = config.esMiniClusterName(); // 获得 147 的 clusterName
String clusterName = config.esMiniClusterName(); // 获得 147 的 clusterName
statisticsTask(clusterName);
end = System.currentTimeMillis();
LOGGER.info("Statistics Task, took:{} ms.",(end - start));
@ -71,13 +71,13 @@ public class StatisticsService {
end = System.currentTimeMillis();
LOGGER.info("Statistics Subject Normal, took:{} ms.",(end - start));
// 如果是【欧莱雅】任务的，得用这个方式统计呀
start = System.currentTimeMillis();
List<BigInteger> subjectIds1 = subjectRepository.findAllOlySubjectIds();
for (BigInteger subjectId: subjectIds1) {
statisticsSubject(subjectId,clusterName);
}
end = System.currentTimeMillis();
LOGGER.info("Statistics Subject OLY, took:{} ms.",(end - start));
// start = System.currentTimeMillis();
// List<BigInteger> subjectIds1 = subjectRepository.findAllOlySubjectIds();
// for (BigInteger subjectId: subjectIds1) {
// statisticsSubject(subjectId,clusterName);
// }
// end = System.currentTimeMillis();
// LOGGER.info("Statistics Subject OLY, took:{} ms.",(end - start));
}
@ -151,7 +151,7 @@ public class StatisticsService {
}else{
siteTodayCount = 0;
}
switch (i) {
switch (i) { //
case 0:
subjectCrawlDatFlagMap.put("keyword", siteCount);
subjectCrawlDataFlagTodayMap.put("keyword", siteTodayCount);
@ -238,6 +238,7 @@ public class StatisticsService {
List<Task> taskList = taskRepository.findAllBydel0();
// 遍历任务List ，根据条件组装ES查询语句去对应的索引下查结果，然后回写到任务表中
for (Task task: taskList) {
System.out.println(" 任务ID ===== " + task);
Long taskId = task.getId().longValue();
String crawlDataFlag = task.getCrawlDataFlag();
String indexNamePre = config.getIndexNamePre();

33
cl_query_data_job/src/main/resources/application.yml
View File

@ -6,9 +6,9 @@ logging:
spring:
datasource:
driver-class-name: com.mysql.jdbc.Driver
username: root
password: Bfd123!@#
url: jdbc:mysql://172.18.1.134:3306/intelligent_crawl?useOldAliasMetadataBehavior=true&characterEncoding=UTF-8&zeroDateTimeBehavior=round
username: crawl
password: crawl
url: jdbc:mysql://172.18.1.181:3306/intelligent_crawl?useOldAliasMetadataBehavior=true&characterEncoding=UTF-8&zeroDateTimeBehavior=round
hikari:
maximum-pool-size: 10
minimum-idle: 1
@ -29,14 +29,14 @@ worker:
## 服务的状态，true 为启动
enable-analysis-producer: false
enable-analysis-consumer: false
enable-statistics-producer: false
enable-statistics-producer: true
enable-query-producer: false
enable-backtrace-producer: false
enable-rw-oly-producer: false
enable-up-load-producer: false
enable-output-producer: false
enable-taskcount-producer: false
enable-alarm-producer: true
enable-alarm-producer: false
## 启动服务的线程数
statistics-producer-thread-count: 1
query-producer-thread-count: 10
@ -64,6 +64,29 @@ worker:
uploadZipPath : /opt/nfsdata/uploadFiles/
indexNamePre : cl_major_
# es-normal:
# name: SQ_Normal
# address: 172.16.10.61:9301
# upper: 2000-01-01
# standby: cl_major_*
# es-reply-source:
# name: SQ_Normal
# address: 172.16.10.61:9301
# upper: 2000-01-01
# standby: cl_major_*
# es-mini:
# name: SQ_Normal
# address: 172.16.10.61:9301
# bulk-thread-count: 5
# bulk-rate: 3
# bulk-size: 100
# es-logstash:
# name: SQ_Normal
# address: 172.16.10.61:9301
# upper: 2021-01-01
# standby: logstash-2021.05.13
es-normal:
name: SQ_Normal_new
address: 172.18.1.134:9301

4
cl_search_api/src/main/java/com/bfd/mf/common/util/es/EsUtils.java
View File

@ -183,7 +183,7 @@ public abstract class EsUtils {
.setQuery(boolQueryBuilder)
.setSearchType(SearchType.DEFAULT)
.setSize(limit)
.setScroll(new TimeValue(20000))
.setScroll(new TimeValue(300000))
.execute()
.actionGet();//注意:首次搜索并不包含数据
}catch (Exception e){
@ -195,7 +195,7 @@ public abstract class EsUtils {
try {
searchResponse = client
.prepareSearchScroll(scrollId)
.setScroll(new TimeValue(20000))
.setScroll(new TimeValue(300000))
.execute()
.actionGet();
} catch (Exception e) {

10
cl_search_api/src/main/java/com/bfd/mf/controller/SearchDataController.java
View File

@ -34,8 +34,6 @@ public class SearchDataController {
@Autowired
private SearchDataService searchDataService;
@Autowired
private SiteRepository siteRepository;
@Autowired
private UpdateService updateService;
@ -58,13 +56,9 @@ public class SearchDataController {
}else if(subjectId.equals("")){ // 没有专题
result.put(ESConstant.ALLDOCNUMBER,0L);
result.put(ESConstant.MONITORLISTS,new ArrayList<>());
}else{ // 专题数据导出
//Integer searchType = queryRequest.getSearchType();
// if(searchType == 0) {
// result = searchDataService.exportDataInSubjectIndexTestGroupBy(queryRequest);
// }else{
}else{
// 专题数据导出
result = searchDataService.exportDataInSubjectIndex(queryRequest);
// }
}
return ResponseWrapper.buildResponse(RTCodeEnum.C_OK, result);
}else {

4
cl_search_api/src/main/resources/application.yml
View File

@ -11,8 +11,8 @@ spring:
datasource:
driver-class-name: com.mysql.jdbc.Driver
username: root
password: bfd123
url: jdbc:mysql://172.26.11.113:3306/intelligent_crawl_item?useOldAliasMetadataBehavior=true&characterEncoding=UTF-8&zeroDateTimeBehavior=round
password: Bfd123!@#
url: jdbc:mysql://172.18.1.134:3306/intelligent_crawl?useOldAliasMetadataBehavior=true&characterEncoding=UTF-8&zeroDateTimeBehavior=round
hikari:
maximum-pool-size: 10

4
logs/sdkclient_error.log
View File

@ -1,4 +0,0 @@
2021-05-22 17:21:45,031 ERROR [com.bfd.mf.job.util.EMailUtils] EMailUtils:sendEmail error. title:{cid=Nreuters} emailList:[jing.du@percent.cn]
2021-05-22 17:21:45,039 ERROR [com.bfd.mf.job.util.EMailUtils] EMailUtils:sendEmail error. title:{cid=Nrusi} emailList:[jing.du@percent.cn]
2021-05-22 17:21:45,045 ERROR [com.bfd.mf.job.util.EMailUtils] EMailUtils:sendEmail error. title:{cid=Nusadefense} emailList:[jing.du@percent.cn]
2021-05-22 17:21:45,053 ERROR [com.bfd.mf.job.util.EMailUtils] EMailUtils:sendEmail error. title:{cid=Nenotrans} emailList:[jing.du@percent.cn]

0
logs/sdkclient_error.log.2021-05-12-14
View File

Write Preview
Loading…
Cancel
Save