maojian
/
analyst_assistant_schedule
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

有知数据流

master
55007 6 months ago
commit
b6c839c638
59 changed files with 6110 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 40
      .classpath
  2. 3
      .gitignore
  3. 23
      .project
  4. 5
      .settings/org.eclipse.core.resources.prefs
  5. 9
      .settings/org.eclipse.jdt.core.prefs
  6. 4
      .settings/org.eclipse.m2e.core.prefs
  7. 1
      README.md
  8. 269
      pom.xml
  9. 29
      src/main/java/com/bfd/analyze/Application.java
  10. 49
      src/main/java/com/bfd/analyze/cache/ConfigCache.java
  11. 42
      src/main/java/com/bfd/analyze/config/BillingConfig.java
  12. 20
      src/main/java/com/bfd/analyze/config/KafkaConfig.java
  13. 43
      src/main/java/com/bfd/analyze/config/ThreadPoolConfig.java
  14. 25
      src/main/java/com/bfd/analyze/config/ZookeeperConfig.java
  15. 115
      src/main/java/com/bfd/analyze/controller/ProcessTriggerController.java
  16. 41
      src/main/java/com/bfd/analyze/entity/AnalystData.java
  17. 149
      src/main/java/com/bfd/analyze/entity/Constants.java
  18. 314
      src/main/java/com/bfd/analyze/handler/DataExecHandler.java
  19. 71
      src/main/java/com/bfd/analyze/mapper/AppsMapper.java
  20. 51
      src/main/java/com/bfd/analyze/mapper/BlueprintMapper.java
  21. 42
      src/main/java/com/bfd/analyze/mapper/ModulesMapper.java
  22. 37
      src/main/java/com/bfd/analyze/mapper/RelationsMapper.java
  23. 47
      src/main/java/com/bfd/analyze/mapper/ScenesMapper.java
  24. 56
      src/main/java/com/bfd/analyze/model/AppsEntity.java
  25. 44
      src/main/java/com/bfd/analyze/model/BlueprintEntity.java
  26. 53
      src/main/java/com/bfd/analyze/model/ModulesEntity.java
  27. 42
      src/main/java/com/bfd/analyze/model/RelationsEntity.java
  28. 59
      src/main/java/com/bfd/analyze/model/ScenesEntity.java
  29. 50
      src/main/java/com/bfd/analyze/monitor/ZookeeperNodeMonitor.java
  30. 75
      src/main/java/com/bfd/analyze/process/ErrorTaskProcess.java
  31. 365
      src/main/java/com/bfd/analyze/process/ResultParseProcess.java
  32. 48
      src/main/java/com/bfd/analyze/process/ResultSendQueue.java
  33. 57
      src/main/java/com/bfd/analyze/process/SaveDataProcess.java
  34. 248
      src/main/java/com/bfd/analyze/process/TaskdispatchProcess.java
  35. 18
      src/main/java/com/bfd/analyze/service/DataService.java
  36. 23
      src/main/java/com/bfd/analyze/service/DistributedLockService.java
  37. 59
      src/main/java/com/bfd/analyze/service/ProcessTriggerService.java
  38. 240
      src/main/java/com/bfd/analyze/service/impl/DataServiceImpl.java
  39. 40
      src/main/java/com/bfd/analyze/service/impl/DistributedLockServiceImpl.java
  40. 637
      src/main/java/com/bfd/analyze/service/impl/ProcessTriggerServiceImpl.java
  41. 79
      src/main/java/com/bfd/analyze/test/CreateElasticsearchIndex.java
  42. 58
      src/main/java/com/bfd/analyze/test/DeleteElasticsearchIndex.java
  43. 59
      src/main/java/com/bfd/analyze/test/HttpClientDeleteExample.java
  44. 81
      src/main/java/com/bfd/analyze/test/IndexDataWithHttpClient.java
  45. 177
      src/main/java/com/bfd/analyze/utils/DateUtil.java
  46. 912
      src/main/java/com/bfd/analyze/utils/DownLoadUtil.java
  47. 408
      src/main/java/com/bfd/analyze/utils/ElasticsearchUtil.java
  48. 27
      src/main/java/com/bfd/analyze/utils/EncryptionUtil.java
  49. 30
      src/main/java/com/bfd/analyze/utils/FileUtil.java
  50. 39
      src/main/java/com/bfd/analyze/utils/JsonUtil.java
  51. 32
      src/main/java/com/bfd/analyze/utils/OtherUtils.java
  52. 69
      src/main/java/com/bfd/analyze/utils/PauseTool.java
  53. 107
      src/main/java/com/bfd/analyze/utils/QueueUtil.java
  54. 48
      src/main/java/com/bfd/analyze/utils/SpringBootKafka.java
  55. 226
      src/main/java/com/bfd/analyze/utils/SqlStatementInterceptor.java
  56. 23
      src/main/java/com/bfd/analyze/utils/ThrowMessageUtil.java
  57. 136
      src/main/resources/application.yml
  58. 36
      src/main/resources/logback-spring.xml
  59. 20
      src/test/java/com/bfd/AppTest.java

40
.classpath
View File

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="optional" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
<attribute name="optional" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

3
.gitignore
View File

@ -0,0 +1,3 @@
/target/
/logs/
/.idea/

23
.project
View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>analyst_assistant_schedule</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>

5
.settings/org.eclipse.core.resources.prefs
View File

@ -0,0 +1,5 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding/<project>=UTF-8

9
.settings/org.eclipse.jdt.core.prefs
View File

@ -0,0 +1,9 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.methodParameters=generate
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8

4
.settings/org.eclipse.m2e.core.prefs
View File

@ -0,0 +1,4 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1

1
README.md
View File

@ -0,0 +1 @@
分析助手应用调度

269
pom.xml
View File

@ -0,0 +1,269 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.4.RELEASE</version>
</parent>
<groupId>com.bfd.analyze</groupId>
<artifactId>analyst_assistant_schedule</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>analyst_assistant_schedule</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/de.codecentric/spring-boot-admin-starter-client -->
<dependency>
<groupId>de.codecentric</groupId>
<artifactId>spring-boot-admin-starter-client</artifactId>
<version>2.2.4</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.8</version>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-test</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework/spring-test -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>5.0.10.RELEASE</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>2.0.17</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.mchange/c3p0 -->
<dependency>
<groupId>com.mchange</groupId>
<artifactId>c3p0</artifactId>
<version>0.9.5.5</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.29</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.9.3</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.jetbrains.kotlin/kotlin-reflect -->
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-reflect</artifactId>
<version>1.6.21</version>
<scope>runtime</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/redis.clients/jedis -->
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>3.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.8.1</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<!-- <dependency> -->
<!-- <groupId>org.apache.kafka</groupId> -->
<!-- <artifactId>kafka-clients</artifactId> -->
<!-- <version>2.6.0</version> -->
<!-- </dependency> -->
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.5</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/p6spy/p6spy -->
<dependency>
<groupId>p6spy</groupId>
<artifactId>p6spy</artifactId>
<version>3.9.0</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
</dependency>
<!--redis-->
<dependency>
<groupId>org.redisson</groupId>
<artifactId>redisson-spring-boot-starter</artifactId>
<version>3.13.6</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>5.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>5.2.0</version>
</dependency>
</dependencies>
<build>
<!-- <pluginManagement> --><!-- lock down plugins versions to avoid using Maven defaults (may be moved
to parent pom) -->
<plugins>
<!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
<plugin>
<artifactId>maven-site-plugin</artifactId>
<version>3.7.1</version>
</plugin>
<plugin>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<!-- spring-boot-maven-plugin插件就是打包spring boot应用的 -->
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<mainClass>com.bfd.analyze.Application</mainClass>
<layout>ZIP</layout>
<includes>
<include>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
</include>
</includes>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.1.1</version>
<executions>
<execution>
<id>copy</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<type>jar</type>
<includeTypes>jar</includeTypes>
<includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/libs</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<!-- </pluginManagement> -->
</build>
</project>

29
src/main/java/com/bfd/analyze/Application.java
View File

@ -0,0 +1,29 @@
package com.bfd.analyze;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.scheduling.annotation.EnableScheduling;
/**
* 主入口
*
* @author jian.mao
* @date 2023年7月4日
* @description
*/
@SpringBootApplication
@MapperScan("com.bfd.analyze.mapper")
@EnableScheduling
@EnableKafka
public class Application {
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}
}

49
src/main/java/com/bfd/analyze/cache/ConfigCache.java
View File

@ -0,0 +1,49 @@
package com.bfd.analyze.cache;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.concurrent.LinkedBlockingDeque;
/**
* @author jian.mao
* @date 2022年11月11日
* @description 静态变量类
*/
@Slf4j
public class ConfigCache {
/**启动条件**/
public static boolean isStart = true;
/*****任务队列*****/
public static LinkedBlockingDeque<Map<String, Object>> taskQueue = null;
/*****结果集队列*****/
public static LinkedBlockingDeque<Map<String, Object>> ResultQueue = null;
/*****持久化到es队列*****/
public static LinkedBlockingDeque<Map<String, Object>> saveDataQueue = null;
/*****错误任务队列*****/
public static LinkedBlockingDeque<Map<String, Object>> errorTaskQueue = null;
/*****周期任务等待队列*****/
public static LinkedBlockingDeque<Map<String, Object>> cycleTaskWaitQueue = null;
/*****等待app任务缓存****/
/*****
public static Map<String, Object> waitTask = new HashMap<String, Object>(1024);
* ****/
public static String resultTopic = null;
/**
* 队列录入任务
* @param queue
* @param task
*/
public static void putQueue(LinkedBlockingDeque<Map<String, Object>> queue,Map<String, Object> task){
//next app 写入队列准备调出
try {
queue.put(task);
} catch (InterruptedException e) {
log.error("队列写入data失败---");
}
}
}

42
src/main/java/com/bfd/analyze/config/BillingConfig.java
View File

@ -0,0 +1,42 @@
package com.bfd.analyze.config;
import java.util.Map;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* @author jian.mao
* @date 2024年9月24日
* @description
*/
@Component
@ConfigurationProperties(prefix = "billing")
public class BillingConfig {
private Map<String, Object> apps;
private String billingUrl;
private int aimodelType;
public Map<String, Object> getApps() {
return apps;
}
public void setApps(Map<String, Object> apps) {
this.apps = apps;
}
public String getBillingUrl() {
return billingUrl;
}
public void setBillingUrl(String billingUrl) {
this.billingUrl = billingUrl;
}
public int getAimodelType() {
return aimodelType;
}
public void setAimodelType(int aimodelType) {
this.aimodelType = aimodelType;
}
}

20
src/main/java/com/bfd/analyze/config/KafkaConfig.java
View File

@ -0,0 +1,20 @@
package com.bfd.analyze.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
/**
* kafka配置类
* @author jian.mao
* @date 2023年7月6日
* @description
*/
@Configuration
public class KafkaConfig {
@Value("${kafka.consumer.topic}")
private String kafkaTopic;
public String getKafkaTopic() {
return kafkaTopic;
}
}

43
src/main/java/com/bfd/analyze/config/ThreadPoolConfig.java
View File

@ -0,0 +1,43 @@
package com.bfd.analyze.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.*;
/**
* @author jian.mao
* @date 2023年3月23日
* @description
*/
@Configuration
public class ThreadPoolConfig {
/** 核心线程数（默认线程数） */
private static final int CORE_POOL_SIZE = 20;
/** 最大线程数 */
private static final int MAX_POOL_SIZE = 50;
/** 允许线程空闲时间（单位：默认为秒） */
private static final int KEEP_ALIVE_TIME = 10;
/** 缓冲队列大小 */
private static final int QUEUE_CAPACITY = 200;
/** 线程池名前缀 */
private static final String THREAD_NAME_PREFIX = "ThreadPool-";
@Bean("taskExecutor")
public ThreadPoolTaskExecutor taskExecutor(){
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(CORE_POOL_SIZE);
executor.setMaxPoolSize(MAX_POOL_SIZE);
executor.setQueueCapacity(QUEUE_CAPACITY);
executor.setKeepAliveSeconds(KEEP_ALIVE_TIME);
executor.setThreadNamePrefix(THREAD_NAME_PREFIX);
// 线程池对拒绝任务的处理策略
// CallerRunsPolicy：由调用线程（提交任务的线程）处理该任务
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
// 初始化
executor.initialize();
return executor;
}
}

25
src/main/java/com/bfd/analyze/config/ZookeeperConfig.java
View File

@ -0,0 +1,25 @@
package com.bfd.analyze.config;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @author jian.mao
* @date 2024年4月16日
* @description
*/
@Configuration
public class ZookeeperConfig {
@Value("${zookeeper.connection-string}")
private String connectionString;
@Bean
public CuratorFramework curatorFramework() {
CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(connectionString, new ExponentialBackoffRetry(1000, 3));
curatorFramework.start();
return curatorFramework;
}
}

115
src/main/java/com/bfd/analyze/controller/ProcessTriggerController.java
View File

@ -0,0 +1,115 @@
package com.bfd.analyze.controller;
import javax.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.curator.framework.CuratorFramework;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import com.bfd.analyze.service.ProcessTriggerService;
/**
* 任务管理接口
*
* @author jian.mao
* @date 2023年9月19日
* @description
*/
@Controller
@RequestMapping("/trigger")
@Slf4j
public class ProcessTriggerController {
@Autowired
private CuratorFramework curatorFramework;
@Resource
private ProcessTriggerService processTriggerService;
@PostMapping("/start")
@ResponseBody
public String processStart(@RequestBody String dataJson) {
String result = processTriggerService.start(dataJson);
return result;
}
@RequestMapping(value = "/hello", method = RequestMethod.GET)
@ResponseBody
public String hello(String param, String token) {
return "123";
}
@PostMapping("/stop")
@ResponseBody
public String processStop(@RequestBody String dataJson) {
String result = processTriggerService.stop(dataJson);
return result;
}
/**
* 场景删除
* @param dataJson
* @return
*/
@PostMapping("/scenesDelete")
@ResponseBody
public String scenesDelete(@RequestBody String dataJson) {
String result = processTriggerService.scenesDelete(dataJson);
return result;
}
/**
* 流程主动触发
* @param dataJson
* @return
*/
@PostMapping("/active")
@ResponseBody
public String activeExec(@RequestBody String dataJson) {
String result = processTriggerService.active(dataJson);
return result;
}
/**
* 暂停
* @param dataJson
* @return
*/
@PostMapping("/pause")
@ResponseBody
public String pause(@RequestBody String dataJson) {
String result = processTriggerService.pause(dataJson);
return result;
}
/**
* 逻辑删除数据
* @param dataJson
* @return
*/
@PostMapping("/delete")
@ResponseBody
public String delete(@RequestBody String dataJson) {
String result = processTriggerService.delete(dataJson);
return result;
}
/**
* 重试
* @param dataJson
* @return
*/
@PostMapping("/retry")
@ResponseBody
public String retry(@RequestBody String dataJson) {
String result = processTriggerService.retry(dataJson);
return result;
}
}

41
src/main/java/com/bfd/analyze/entity/AnalystData.java
View File

@ -0,0 +1,41 @@
package com.bfd.analyze.entity;
import lombok.Data;
import java.util.List;
import java.util.Map;
/**
* 最终存储的数据体实体类
*
* @author jian.mao
* @date 2023年7月11日
* @description
*/
@Data
public class AnalystData {
private String created;
private String lastEdit;
private Integer appId;
private Integer transferId;
private String appName;
private String appCode;
private String appDescribe;
private String module;
private String result;
private List<Map<String, Object>> nextAppId;
private Boolean startTag;
private Integer scenesId;
private String businessKey;
private String dataId;
private String dataProcessId;
private List<Map<String, Object>> variable;
private Integer del;
private Integer status;
private String message;
private Integer version;
private String dispatchId;
private List<String> parentId;
private Integer trace;
}

149
src/main/java/com/bfd/analyze/entity/Constants.java
View File

@ -0,0 +1,149 @@
package com.bfd.analyze.entity;
/**
* 常量实体类
* @author jian.mao
* @date 2022年11月15日
* @description
*/
public class Constants {
/*************************蓝图常量key名称*********************************/
public final static String SCHEDULING = "scheduling";
public final static String TYPE = "type";
public final static String INTERVAL = "interval";
public final static String CREATED = "created";
public final static String LAST_EDIT = "last_edit";
public final static String BLUEPRINT_ID = "blueprint_id";
public final static String BLUEPRINT_NAME = "name";
public final static String SCENARIO = "scenario";
public final static String AUTOCOMMITTRIGGERLAST = "autoCommitTriggerLast";
public final static String FRESHVARIABLES = "freshVariables";
public final static String AUTOCOMMIT = "autoCommit";
public final static String MAXERRORS = "maxErrors";
public final static String DATALOSS = "dataloss";
public final static String POSITION = "position";
public final static String SCENES_ID = "scenes_id";
public final static String COVER = "cover";
public final static String MULTI_BRANCH = "multi_branch";
public final static String SINGLE = "single";
/**已重试次数**/
public final static String ERROR_TIME = "error_time";
public final static String PREVIOUS_RESULT = "previous_result";
/****数据id*****/
public final static String BUSINESSKEY = "businessKey";
/*************************metadata常量key名称*********************************/
public final static String OUTPUT = "output";
public final static String LABEL_COL = "label_col";
public final static String LABEL = "label";
public final static String INPUT = "input";
public final static String USER = "user";
public final static String ADMIN = "admin";
public final static String ADDRESS = "address";
public final static String DATASOURCE = "datasource";
public final static String INDEX = "index";
/*************************app常量key名称*********************************/
public final static String APPS = "apps";
public final static String APP_ID = "id";
public final static String TRANSFER_ID = "transfer_id";
public final static String MODULE = "module";
public final static String VERSION = "version";
public final static String DATA = "data";
public final static String APP_NAME = "name";
public final static String DESCRIBE = "describe";
public final static String APP_CODE = "app_code";
public final static String NEXT_APP_ID = "next_app_id";
public final static String EDGE_ID = "edge_id";
public final static String START_ID = "start_id";
public final static String END_ID = "end_id";
public static final String DATAID = "dataId";
public static final String NOTE = "note";
public final static String WAIT_CONDITION = "wait_condition";
public final static String START_TAG = "start_tag";
public final static String RELATIONS = "relations";
public static final String SCHEDULING_TYPE = "scheduling_type";
public static final String NEXT_SCHEDULING_TIME = "next_scheduling_time";
public static final String FLAG_DATA = "flag_data";
/*************************module类型*********************************/
public final static String FILE = "file";
public final static String OCR = "OCR";
public final static String FILTER = "Filter";
public final static String CHATGPT = "ChatGPT";
public final static String MYSQL = "mysql";
/*************************other类型*********************************/
public final static String UNDERLINE = "_";
public final static String RESULT_TOPIC = null;
public static final String EMPTY = "";
public static final String HTTP = "http";
public static final String REQUEST_ERROR_MESSAGE = "Download failed error is";
public static final String REQUEST_RESULT = "result";
public static final String REQUEST_RESULT_RESULTS = "results";
public static final String MAP_TYPE = "Map";
public static final String LIST_TYPE = "List";
public static final String STRING_TYPE = "String";
public static final String DOCUMENT_TYPE = "doc";
public static final String FILTER_ZH = "过滤器";
public static final String JSON_SELE_SYMBOL = "$.";
public static final String LEFT_BRACKETS = "[";
public static final String RIGTH_BRACKETS = "]";
public static final String ISLAST = "isLast";
/****是否是扩散型应用标识key*****/
public static final String IS_DIFFUSION = "is_diffusion";
/************************redis*************************************/
public static final String LOCK_KEY = "myLock";
public static final long LOCK_EXPIRE_TIME = 300000;
public static final String CODE = "code";
public static final String MESSAGE = "message";
public static final String FORM = "form";
public static final String FIELD = "field";
public static final String SOURCE_DATA_ID = "source_data_id";
public static final String WILDCARD = "*";
public static final String DATAPROCESSID = "dataProcessId";
public static final String HITS = "hits";
public static final String ES_SOURCE = "_source";
public static final String APPCODE = "appCode";
public static final String CREATEUSERID = "createUserId";
public static final long LOCKTIME = 10L;
public static final String IS_ACTIVE = "is_active";
public static final String DATAIDS = "dataIds";
public static final String SEARCHFIELDS = "searchFields";
public static final String DEL = "del";
public static final String UPDATED = "updated";
public static final String STATUS = "status";
public static final String NEXTAPPID = "nextAppId";
public static final String SCENESID = "scenesId";
/**聚合类应用，多个数据到此应用只产生一条结果**/
public static final String POLYMERIZATION = "polymerization";
/*************************接口动作************************************/
public static final String START = "start";
public static final String STOP = "stop";
public static final String DELETE = "delete";
public static final String MATCH = "match";
public static final String TERMS = "terms";
public static final String TERM = "term";
public static final String DISPATCHID = "dispatchId";
public static final String PARENTID = "parentId";
public static final String POINT_KEYWORD = ".keyword";
public static final String DEFAULTCOMPLETION = "defaultCompletion";
public static final String ERROR = "error";
public static final String PLANCODE = "planCode";
public static final String SCENES_NAME = "scenes_name";
public static final String CDA_ = "cda_";
public static final String MODULE_ID = "module_id";
public static final String MODELTYPE = "modelType";
public static final String BILLINGSUFFIX = "/api/pay/api/publicTransaction/spend";
public static final String TRACE = "trace";
}

314
src/main/java/com/bfd/analyze/handler/DataExecHandler.java
View File

@ -0,0 +1,314 @@
package com.bfd.analyze.handler;
import com.alibaba.fastjson.JSONObject;
import com.bfd.analyze.cache.ConfigCache;
import com.bfd.analyze.entity.Constants;
import com.bfd.analyze.mapper.AppsMapper;
import com.bfd.analyze.mapper.BlueprintMapper;
import com.bfd.analyze.mapper.ModulesMapper;
import com.bfd.analyze.mapper.RelationsMapper;
import com.bfd.analyze.process.ErrorTaskProcess;
import com.bfd.analyze.process.ResultParseProcess;
import com.bfd.analyze.process.SaveDataProcess;
import com.bfd.analyze.process.TaskdispatchProcess;
import com.bfd.analyze.utils.DateUtil;
import com.bfd.analyze.utils.FileUtil;
import com.bfd.analyze.utils.PauseTool;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.curator.framework.CuratorFramework;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.core.annotation.Order;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingDeque;
/**
* 调度执行入口
*
* @author jian.mao
* @date 2023年7月4日
* @description
*/
@Component
@Order(value = 1)
@Slf4j
public class DataExecHandler implements ApplicationRunner {
@Resource
private BlueprintMapper blueprintMapper;
@Resource
private AppsMapper appsMapper;
@Resource
private ModulesMapper modulesMapper;
@Resource
private RelationsMapper relationsMapper;
@Autowired
private TaskdispatchProcess taskdispatchProcess;
@Autowired
private ResultParseProcess resultParseProcess;
@Autowired
private SaveDataProcess saveDataProcess;
@Autowired
private ErrorTaskProcess errorTaskProcess;
@Value("${task.task-queue-path}")
private String taskPath;
@Value("${task.result-queue-path}")
private String resultPath;
@Value("${task.saveData-queue-path}")
private String saveDataPath;
@Value("${task.errorTask-queue-path}")
private String errorTaskPath;
@Value("${task.dispatch-thread-num}")
private Integer dispatchThreadNum;
@Value("${task.results-thread-num}")
private Integer resultsThreadNum;
@Value("${task.save-thread-num}")
private Integer saveThreadNum;
@Value("${task.queue-size}")
private Integer queueSize;
@Autowired
private StringRedisTemplate stringRedisTemplate;
@Autowired
private CuratorFramework curatorFramework;
@Autowired
PauseTool pauseTool;
/**
@Autowired
private DistributedLockService distributedLockService;
@Autowired
private StringRedisTemplate stringRedisTemplate;
**/
/********************************************************start***************************************************************/
/**
* 执行入口
*/
@Override
public void run(ApplicationArguments args) {
//设置队列长度
if(queueSize == null || queueSize == 0){
queueSize = 10000;
}
/*****任务队列*****/
ConfigCache.taskQueue = new LinkedBlockingDeque<Map<String,Object>>(queueSize);
/*****结果集队列*****/
ConfigCache.ResultQueue = new LinkedBlockingDeque<Map<String,Object>>(queueSize);
/*****持久化到es队列*****/
ConfigCache.saveDataQueue = new LinkedBlockingDeque<Map<String,Object>>(queueSize);
/*****错误任务队列*****/
ConfigCache.errorTaskQueue = new LinkedBlockingDeque<Map<String,Object>>();
/*****周期任务等待队列*****/
ConfigCache.cycleTaskWaitQueue = new LinkedBlockingDeque<Map<String,Object>>();
//等待es连接成功在注入es连接对象
DateUtil.sleep(5000);
if(dispatchThreadNum == null || dispatchThreadNum == 0){
dispatchThreadNum = 3;
}
if(resultsThreadNum == null || resultsThreadNum == 0){
resultsThreadNum = 3;
}
if(saveThreadNum == null || saveThreadNum == 0){
saveThreadNum = 3;
}
for (int i = 0; i < dispatchThreadNum; i++) {
log.info("启动发送线程---");
new Thread(taskdispatchProcess).start();
}
for (int i = 0; i < resultsThreadNum; i++) {
log.info("启动调回线程---");
new Thread(resultParseProcess).start();
}
for (int i = 0; i < saveThreadNum; i++) {
log.info("启动数据保存线程---");
new Thread(saveDataProcess).start();
}
log.info("启动错误任务处理线程---");
new Thread(errorTaskProcess).start();
//启动加载缓存任务
readTask(taskPath,ConfigCache.taskQueue);
readTask(resultPath,ConfigCache.ResultQueue);
readTask(saveDataPath,ConfigCache.saveDataQueue);
readTask(errorTaskPath,ConfigCache.errorTaskQueue);
//初始化 redis 场景版本缓存
pauseTool.initializeRedisCache(stringRedisTemplate);
//停止处理
waitDown();
}
public static void readTask(String path,LinkedBlockingDeque<Map<String, Object>> queue){
File file = new File(path);
if(file.exists()){
List<String> tasks = null;
try {
tasks = FileUtils.readLines(file,"UTF-8");
} catch (IOException e) {
e.printStackTrace();
}
for (String taskStr : tasks) {
Map<String, Object> task = JSONObject.parseObject(taskStr);
try {
queue.put(task);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
file.delete();
}
}
/**
* 查询表，组装任务
*/
@Scheduled(cron = "*/10 * * * * ?")
private void scanTask() {
/**************************************************old start********************************************************/
//获取分布式锁
// if(distributedLockService.acquireLock()){
// try {
// //打印队列长度
// log.info("发送队列长度：{}",ConfigCache.taskQueue.size());
// log.info("调回队列长度：{}",ConfigCache.ResultQueue.size());
// log.info("保存队列长度：{}",ConfigCache.saveDataQueue.size());
// log.info("错误队列长度：{}",ConfigCache.errorTaskQueue.size());
// //redis分布式锁测试
// log.info("我已占有分布式锁------------------------------------------{}",distributedLockService.acquireLock());
//
//
// } catch (Throwable e) {
// log.error("扫描任务未知错误----",e);
// }finally{
// //释放锁
// distributedLockService.releaseLock();
// }
// }else{
// log.info("任务扫描锁未抢占成功------");
// }
/**************************************************old end********************************************************/
try {
//打印队列长度
log.info("发送队列长度：{}", ConfigCache.taskQueue.size());
log.info("调回队列长度：{}", ConfigCache.ResultQueue.size());
log.info("保存队列长度：{}", ConfigCache.saveDataQueue.size());
log.info("错误队列长度：{}", ConfigCache.errorTaskQueue.size());
log.info("周期等待任务队列长度：{}", ConfigCache.cycleTaskWaitQueue.size());
while (ConfigCache.cycleTaskWaitQueue.size() > 0) {
Map<String, Object> task = ConfigCache.cycleTaskWaitQueue.poll();
long currentTimeMillis = System.currentTimeMillis();
long nextSchedulingTime = (long) task.get(Constants.NEXT_SCHEDULING_TIME);
if (currentTimeMillis > nextSchedulingTime) {
//符合调出
ConfigCache.taskQueue.put(task);
log.info("周期任务调出：{}", JSONObject.toJSONString(task));
} else {
//不符合调出时间，放回队列，进行等待
ConfigCache.cycleTaskWaitQueue.put(task);
}
}
} catch (Throwable e) {
log.error("扫描任务未知错误----", e);
}
}
/**************************************************stop***********************************************************************/
/**
* 结束触发钩子
*/
public void waitDown() {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
// 停止线程
ConfigCache.isStart = false;
//释放分布式锁
// distributedLockService.releaseLock();
log.info("stop-------");
writeTsskToFile();
}
});
}
/**
* 任务持久化到硬盘
*/
public void writeTsskToFile(){
while(true){
if(ConfigCache.taskQueue.size() > 0 ){
try {
Map<String, Object> task = ConfigCache.taskQueue.take();
FileUtil.writeFile(taskPath, JSONObject.toJSONString(task));
} catch (InterruptedException e) {
e.printStackTrace();
}
}else{
log.info("taskQueue write is file end");
break;
}
}
while(true){
if(ConfigCache.ResultQueue.size() > 0 ){
try {
Map<String, Object> task = ConfigCache.ResultQueue.take();
FileUtil.writeFile(resultPath, JSONObject.toJSONString(task));
} catch (InterruptedException e) {
e.printStackTrace();
}
}else{
log.info("ResultQueue write is file end");
break;
}
}
while(true){
if(ConfigCache.saveDataQueue.size() > 0 ){
try {
Map<String, Object> task = ConfigCache.saveDataQueue.take();
FileUtil.writeFile(saveDataPath, JSONObject.toJSONString(task));
} catch (InterruptedException e) {
e.printStackTrace();
}
}else{
log.info("saveDataQueue write is file end");
break;
}
}
while(true){
if(ConfigCache.errorTaskQueue.size() > 0 ){
try {
Map<String, Object> task = ConfigCache.errorTaskQueue.take();
FileUtil.writeFile(errorTaskPath, JSONObject.toJSONString(task));
} catch (InterruptedException e) {
e.printStackTrace();
}
}else{
log.info("errorTaskQueue write is file end");
break;
}
}
}
}

71
src/main/java/com/bfd/analyze/mapper/AppsMapper.java
View File

@ -0,0 +1,71 @@
package com.bfd.analyze.mapper;
import com.bfd.analyze.model.AppsEntity;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Result;
import org.apache.ibatis.annotations.Results;
import org.apache.ibatis.annotations.Select;
import java.util.List;
/**
* APP节点mapper接口
* @author jian.mao
* @date 2023年7月5日
* @description
*/
public interface AppsMapper {
@Select({"select * from apps where scenes_id = #{scenesId} and del = 0"})
@Results({
@Result(column = "id",property = "id"),
@Result(column = "create_user_id",property = "createUserId"),
@Result(column = "create_user",property = "createUser"),
@Result(column = "update_user_id",property = "updateUserId"),
@Result(column = "update_user",property = "updateUser"),
@Result(column = "create_time",property = "createTime"),
@Result(column = "update_time",property = "updateTime"),
@Result(column = "del",property = "del"),
@Result(column = "app_code",property = "appCode"),
@Result(column = "transfer_id",property = "transferId"),
@Result(column = "scenes_id",property = "scenesId"),
@Result(column = "module_id",property = "moduleId"),
@Result(column = "name",property = "name"),
@Result(column = "describe",property = "describe"),
@Result(column = "input",property = "input"),
@Result(column = "output",property = "output"),
@Result(column = "flag_app",property = "flagApp"),
@Result(column = "flag_data",property = "flagData"),
@Result(column = "scheduling_type",property = "schedulingType"),
@Result(column = "wait_condition",property = "waitCondition")
})
public List<AppsEntity> getAppByscenesId(@Param("scenesId") Integer scenesId);
@Select({"select * from apps where app_code = #{appCode}"})
@Results({
@Result(column = "id",property = "id"),
@Result(column = "create_user_id",property = "createUserId"),
@Result(column = "create_user",property = "createUser"),
@Result(column = "update_user_id",property = "updateUserId"),
@Result(column = "update_user",property = "updateUser"),
@Result(column = "create_time",property = "createTime"),
@Result(column = "update_time",property = "updateTime"),
@Result(column = "del",property = "del"),
@Result(column = "app_code",property = "appCode"),
@Result(column = "transfer_id",property = "transferId"),
@Result(column = "scenes_id",property = "scenesId"),
@Result(column = "module_id",property = "moduleId"),
@Result(column = "name",property = "name"),
@Result(column = "describe",property = "describe"),
@Result(column = "input",property = "input"),
@Result(column = "output",property = "output"),
@Result(column = "flag_app",property = "flagApp"),
@Result(column = "flag_data",property = "flagData"),
@Result(column = "scheduling_type",property = "schedulingType"),
@Result(column = "wait_condition",property = "waitCondition")
})
public AppsEntity getAppByAppCode(@Param("appCode") String appCode);
}

51
src/main/java/com/bfd/analyze/mapper/BlueprintMapper.java
View File

@ -0,0 +1,51 @@
package com.bfd.analyze.mapper;
import java.util.List;
import org.apache.ibatis.annotations.Result;
import org.apache.ibatis.annotations.Results;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import com.bfd.analyze.model.BlueprintEntity;
/**
* 蓝图mapper接口
* @author jian.mao
* @date 2023年7月5日
* @description
*/
public interface BlueprintMapper {
@Select({"select * from blueprint where status = 1 and (next_scheduling_time is null or next_scheduling_time < now())"})
@Results({
@Result(column = "blueprint_id",property = "blueprintId"),
@Result(column = "scenes_id",property = "scenesId"),
@Result(column = "name",property = "name"),
@Result(column = "scheduling_type",property = "schedulingType"),
@Result(column = "scheduling_interval",property = "schedulingInterval"),
@Result(column = "auto_commit_trigger_last",property = "autoCommitTriggerLast"),
@Result(column = "dataloss",property = "dataloss"),
@Result(column = "max_errors",property = "maxErrors"),
@Result(column = "auto_commit",property = "autoCommit"),
@Result(column = "fresh_variables",property = "freshVariables"),
@Result(column = "created",property = "created"),
@Result(column = "last_edit",property = "lastEdit"),
@Result(column = "user",property = "user"),
@Result(column = "multi_branch",property = "multiBranch"),
})
public List<BlueprintEntity> getAll();
/**
* 单次任务
* @param blueprintEntity
*/
@Update("update blueprint set status = 3 where blueprint_id = #{blueprintId}")
public void updateSingleBlueprintById(BlueprintEntity blueprintEntity);
@Update("update blueprint set next_scheduling_time = date_add(now(),interval #{schedulingInterval} second) where blueprint_id = #{blueprintId}")
public void updateCycleBlueprintById(BlueprintEntity blueprintEntity);
}

42
src/main/java/com/bfd/analyze/mapper/ModulesMapper.java
View File

@ -0,0 +1,42 @@
package com.bfd.analyze.mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Result;
import org.apache.ibatis.annotations.Results;
import org.apache.ibatis.annotations.Select;
import com.bfd.analyze.model.ModulesEntity;
/**
* APP节点mapper接口
* @author jian.mao
* @date 2023年7月5日
* @description
*/
public interface ModulesMapper {
@Select({"select * from modules where id = #{moduleId}"})
@Results({
@Result(column = "id",property = "id"),
@Result(column = "create_user_id",property = "createUserId"),
@Result(column = "create_user",property = "createUser"),
@Result(column = "update_user_id",property = "updateUserId"),
@Result(column = "update_user",property = "updateUser"),
@Result(column = "create_time",property = "createTime"),
@Result(column = "update_time",property = "updateTime"),
@Result(column = "del",property = "del"),
@Result(column = "name",property = "name"),
@Result(column = "describe",property = "describe"),
@Result(column = "logo",property = "logo"),
@Result(column = "category_id",property = "categoryId"),
@Result(column = "params",property = "params"),
@Result(column = "output",property = "output"),
@Result(column = "output_example",property = "outputExample"),
@Result(column = "flag_release",property = "flagRelease"),
@Result(column = "flag_start",property = "flagStart"),
@Result(column = "url_start",property = "urlStart")
})
public ModulesEntity getModuleByModuleId(@Param("moduleId") Integer moduleId);
}

37
src/main/java/com/bfd/analyze/mapper/RelationsMapper.java
View File

@ -0,0 +1,37 @@
package com.bfd.analyze.mapper;
import com.bfd.analyze.model.RelationsEntity;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Result;
import org.apache.ibatis.annotations.Results;
import org.apache.ibatis.annotations.Select;
import java.util.List;
/**
* APP节点mapper接口
* @author jian.mao
* @date 2023年7月5日
* @description
*/
public interface RelationsMapper {
@Select({"select * from relations where scenes_id = #{scenesId} and del = 0"})
@Results({
@Result(column = "id",property = "id"),
@Result(column = "create_user_id",property = "createUserId"),
@Result(column = "create_user",property = "createUser"),
@Result(column = "update_user_id",property = "updateUserId"),
@Result(column = "update_user",property = "updateUser"),
@Result(column = "create_time",property = "createTime"),
@Result(column = "update_time",property = "updateTime"),
@Result(column = "del",property = "del"),
@Result(column = "scenes_id",property = "scenesId"),
@Result(column = "start_id",property = "startId"),
@Result(column = "end_id",property = "endId"),
@Result(column = "start_code",property = "startCode"),
@Result(column = "end_code",property = "endCode")
})
public List<RelationsEntity> getRelationsByscenesId(@Param("scenesId") Integer scenesId);
}

47
src/main/java/com/bfd/analyze/mapper/ScenesMapper.java
View File

@ -0,0 +1,47 @@
package com.bfd.analyze.mapper;
import java.util.List;
import org.apache.ibatis.annotations.Result;
import org.apache.ibatis.annotations.Results;
import org.apache.ibatis.annotations.Select;
import com.bfd.analyze.model.ScenesEntity;
/**
* 蓝图mapper接口
* @author jian.mao
* @date 2023年7月5日
* @description
*/
public interface ScenesMapper {
@Select({"select * from scenes where id = #{scenesId}"})
@Results({
@Result(column = "id",property = "id"),
@Result(column = "create_user_id",property = "createUserId"),
@Result(column = "create_user",property = "createUser"),
@Result(column = "update_user_id",property = "updateUserId"),
@Result(column = "update_user",property = "updateUser"),
@Result(column = "create_time",property = "createTime"),
@Result(column = "update_time",property = "updateTime"),
@Result(column = "del",property = "del"),
@Result(column = "scenes_name",property = "scenesName"),
@Result(column = "scenes_describe",property = "scenesDescribe"),
@Result(column = "version",property = "version"),
@Result(column = "category_id",property = "categoryId"),
@Result(column = "flag_release",property = "flagRelease"),
@Result(column = "flag_owner",property = "flagOwner"),
@Result(column = "status",property = "status"),
@Result(column = "diagram",property = "diagram"),
@Result(column = "original_scenes_id",property = "originalScenesId"),
@Result(column = "source_id",property = "sourceId"),
@Result(column = "source_data_id",property = "sourceDataId"),
@Result(column = "multi_branch",property = "multiBranch"),
@Result(column = "flag_data",property = "flagData"),
})
public ScenesEntity getscenesById(Integer scenesId);
}

56
src/main/java/com/bfd/analyze/model/AppsEntity.java
View File

@ -0,0 +1,56 @@
package com.bfd.analyze.model;
import lombok.Data;
import java.time.LocalDateTime;
/**
* app节点实体类
*
* @author jian.mao
* @date 2023年7月5日
* @description
*/
@Data
public class AppsEntity {
/****主键****/
private Integer id;
/****创建人id****/
private String createUserId;
/****创建人姓名****/
private String createUser;
/****更新人id****/
private String updateUserId;
/****更新人姓名****/
private String updateUser;
/****创建时间****/
private LocalDateTime createTime;
/****更新时间****/
private LocalDateTime updateTime;
/****删除标识0-未删除，1-已删除****/
private Integer del;
/*** 流程编码前端生成uuid，输出结果与输入配置均使用此项 ***/
private String appCode;
/*** 流转id ***/
private Integer transferId;
/*** 场景id ***/
private Integer scenesId;
/*** 模块id对应定制化应用 ***/
private Integer moduleId;
/*** 节点操作名称 ***/
private String name;
/*** 节点描述 ***/
private String describe;
/*** 输出 ***/
private String output;
/*** 输入 ***/
private String input;
/*** 应用可见0-否 1-是 ***/
private Integer flagApp;
/*** 数据可见0-否 1-是 ***/
private Integer flagData;
/****调出模式****/
private Integer schedulingType;
/****等待节点****/
private String waitCondition;
}

44
src/main/java/com/bfd/analyze/model/BlueprintEntity.java
View File

@ -0,0 +1,44 @@
package com.bfd.analyze.model;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import lombok.Data;
/**
* 蓝图实体类
* @author jian.mao
* @date 2023年7月5日
* @description
*/
@Data
public class BlueprintEntity {
/***蓝图id***/
private Integer blueprintId;
/***场景id***/
private Integer scenesId;
/***蓝图名称***/
private String name;
/***调度类型***/
private String schedulingType;
/***周期***/
private Integer schedulingInterval;
/***蓝图是否最后提交***/
private Integer autoCommitTriggerLast;
/***蓝图状态，成功\失败***/
private Integer dataloss;
/***重试次数***/
private Integer maxErrors;
/***自动提交***/
private Integer autoCommit;
/*** ***/
private Integer freshVariables;
/***创建时间***/
private LocalDateTime created;
/***最后修改时间***/
private Timestamp lastEdit;
/*** ***/
private String user;
/***单、多分支标识***/
public Integer multiBranch;
}

53
src/main/java/com/bfd/analyze/model/ModulesEntity.java
View File

@ -0,0 +1,53 @@
package com.bfd.analyze.model;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 模块实体类
*
* @author jian.mao
* @date 2023年7月5日
* @description
*/
@Data
public class ModulesEntity {
/****主键****/
private Integer id;
/****创建人id****/
private String createUserId;
/****创建人姓名****/
private String createUser;
/****更新人id****/
private String updateUserId;
/****更新人姓名****/
private String updateUser;
/****创建时间****/
private LocalDateTime createTime;
/****更新时间****/
private LocalDateTime updateTime;
/****删除标识0-未删除，1-已删除****/
private Integer del;
/****名称****/
private String name;
/****描述****/
private String describe;
/****logo****/
private String logo;
/****应用分类category.id****/
private Integer categoryId;
/****应用配置仅存储应用个性化固定的配置****/
private String params;
/****输出项****/
private String output;
/****输出样例****/
private String outputExample;
/****发布状态0-未发布 1-已发布****/
private Integer flagRelease;
/****是否可作为起始节点0-不可以 1-可以****/
private Integer flagStart;
/**** 启动url当为起始节点时，流程启动会调用此接口****/
private String urlStart;
}

42
src/main/java/com/bfd/analyze/model/RelationsEntity.java
View File

@ -0,0 +1,42 @@
package com.bfd.analyze.model;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 关系实体类
*
* @author jian.mao
* @date 2023年7月5日
* @description
*/
@Data
public class RelationsEntity {
/****主键****/
private Integer id;
/****创建人id****/
private String createUserId;
/****创建人姓名****/
private String createUser;
/****更新人id****/
private String updateUserId;
/****更新人姓名****/
private String updateUser;
/****创建时间****/
private LocalDateTime createTime;
/****更新时间****/
private LocalDateTime updateTime;
/****删除标识0-未删除，1-已删除****/
private Integer del;
/****场景idscenes.id****/
private String scenesId;
/****起始节点app.id****/
private String startId;
/****结束节点app.id****/
private String endId;
/****起始节点codeapp.app_code****/
private String startCode;
/****结束节点codeapp.app_code****/
private String endCode;
}

59
src/main/java/com/bfd/analyze/model/ScenesEntity.java
View File

@ -0,0 +1,59 @@
package com.bfd.analyze.model;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 场景实体类
*
* @author jian.mao
* @date 2023年7月5日
* @description
*/
@Data
public class ScenesEntity {
/****主键****/
private Integer id;
/****创建人id****/
private String createUserId;
/****创建人姓名****/
private String createUser;
/****更新人id****/
private String updateUserId;
/****更新人姓名****/
private String updateUser;
/****创建时间****/
private LocalDateTime createTime;
/****更新时间****/
private LocalDateTime updateTime;
/****删除标识0-未删除，1-已删除****/
private Integer del;
/****场景名称****/
private String scenesName;
/****场景描述****/
private String scenesDescribe;
/****场景版本复制的新场景重置版本****/
private Integer version;
/****场景分类category.id****/
private Integer categoryId;
/****发布状态0-未发布 1-已发布****/
private Integer flagRelease;
/****场景归属1-公开的 2-私有的****/
private Integer flagOwner;
/****流程状态1-待运行 2-运行中 3-已完成****/
private Integer status;
/****流程画布****/
private String diagram;
/****原始场景id用于场景复制****/
private Integer originalScenesId;
/****来源0-内部 1-采集平台****/
private Integer sourceId;
/****外部主键****/
private String sourceDataId;
/****单、多分支****/
private Integer multiBranch;
/****存储数据开关****/
private Integer flagData;
}

50
src/main/java/com/bfd/analyze/monitor/ZookeeperNodeMonitor.java
View File

@ -0,0 +1,50 @@
package com.bfd.analyze.monitor;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.NodeCache;
import org.apache.curator.framework.recipes.cache.NodeCacheListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
/**
* @author jian.mao
* @date 2024年4月17日
* @description
*
* @Component
*/
public class ZookeeperNodeMonitor {
@Autowired
private CuratorFramework curatorFramework;
@Value("${zookeeper.publish-node}")
private String nodePath;
@PostConstruct
public void init() {
try {
// 创建节点监听器
NodeCache nodeCache = new NodeCache(curatorFramework, nodePath);
nodeCache.start();
// 监听节点变化
nodeCache.getListenable().addListener(new NodeCacheListener() {
@Override
public void nodeChanged() throws Exception {
byte[] data = nodeCache.getCurrentData().getData();
String nodeData = new String(data);
System.out.println("Node data changed: " + nodeData);
// 在这里处理节点数据变化的逻辑，比如通知其他组件或者执行相应的操作
}
});
} catch (Exception e) {
e.printStackTrace();
// 异常处理
}
}
}

75
src/main/java/com/bfd/analyze/process/ErrorTaskProcess.java
View File

@ -0,0 +1,75 @@
package com.bfd.analyze.process;
import com.alibaba.fastjson.JSONObject;
import com.bfd.analyze.cache.ConfigCache;
import com.bfd.analyze.entity.Constants;
import com.bfd.analyze.utils.FileUtil;
import com.bfd.analyze.utils.PauseTool;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.Map;
/**
* 错误任务处理线程
* @author jian.mao
* @date 2023年8月15日
* @description
*/
@Component
@Slf4j
public class ErrorTaskProcess implements Runnable {
@Value("${task.error-task-path}")
private String path;
@Value("${task.error-time}")
private Integer maxErrorTime;
@Autowired
PauseTool pauseTool;
@Override
public void run() {
while(ConfigCache.isStart){
Map<String, Object> task = null;
try {
task = ConfigCache.errorTaskQueue.take();
Integer scense_id = (Integer) task.get(Constants.SCENES_ID);
Integer version = (Integer) task.get(Constants.VERSION);
if (!pauseTool.check(scense_id,version)) {
log.info("暂停任务：{}",JSONObject.toJSONString(task));
continue;
}
errorTaskDealWith(task);
} catch (Exception e) {
log.error("错误任务处理失败---",e);
}
}
}
/**
* 失败任务处理
* @param errorTask
*/
private void errorTaskDealWith(Map<String, Object> errorTask){
if(errorTask.containsKey(Constants.ERROR_TIME)){
Integer errorTime = (Integer) errorTask.get(Constants.ERROR_TIME);
if(errorTime < maxErrorTime){
//没到重试上限
errorTask.put(Constants.ERROR_TIME, (errorTime+1));
//失败任务写kafka重新流转
//直接放回队列会出现死锁情况
ConfigCache.putQueue(ConfigCache.taskQueue, errorTask);
}else{
//已达上限---写入错误文件
FileUtil.writeFile(path, JSONObject.toJSONString(errorTask));
}
}else{
//第一次重试
errorTask.put(Constants.ERROR_TIME, 1);
//失败任务写kafka重新流转
ConfigCache.putQueue(ConfigCache.taskQueue, errorTask);
}
}
}

365
src/main/java/com/bfd/analyze/process/ResultParseProcess.java
View File

@ -0,0 +1,365 @@
package com.bfd.analyze.process;
import cn.hutool.core.lang.UUID;
import com.alibaba.fastjson.JSONObject;
import com.bfd.analyze.cache.ConfigCache;
import com.bfd.analyze.config.BillingConfig;
import com.bfd.analyze.entity.Constants;
import com.bfd.analyze.utils.DownLoadUtil;
import com.bfd.analyze.utils.ElasticsearchUtil;
import com.bfd.analyze.utils.EncryptionUtil;
import com.bfd.analyze.utils.JsonUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.HashOperations;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
/**
* 任务结果处理
* @author jian.mao
* @date 2023年7月6日
* @description
*/
@Component
@Slf4j
public class ResultParseProcess implements Runnable {
@Autowired
private StringRedisTemplate stringRedisTemplate;
@Value("${elasticsearch.index-name}")
private String indexName;
@Value("${elasticsearch.cluster-nodes}")
private String nodes;
@Value("${elasticsearch.username}")
private String username;
@Value("${elasticsearch.password}")
private String password;
@Autowired
private BillingConfig billingConfig;
@Override
public void run() {
// TODO Auto-generated method stub
while(ConfigCache.isStart){
try {
Map<String, Object> result = null;
try {
result = ConfigCache.ResultQueue.take();
} catch (InterruptedException e) {
log.error("获取调回结果失败---",e);
}
//数据新增内部唯一标识（二次推送不用新增数据id）
String dataId = UUID.randomUUID().toString();
if(!result.containsKey(Constants.DATAID)){
result.put(Constants.DATAID, dataId);
}else if(!result.containsKey(Constants.COVER)){
result.put(Constants.DATAID, dataId);
}
//数据新增主体id 从流程起始到结尾都留存的id
if(!result.containsKey(Constants.DATAPROCESSID)){
result.put(Constants.DATAPROCESSID, UUID.randomUUID().toString());
}
//扩散类型的应用存在parentId字段
if(result.containsKey(Constants.IS_DIFFUSION) && (boolean)result.get(Constants.IS_DIFFUSION)){
if(!result.containsKey(Constants.PARENTID)){
//不包含父节点，第一个扩散型应用
List<String> parentIds = new ArrayList<String>();
parentIds.add(dataId);
result.put(Constants.PARENTID, parentIds);
}else{
//第一个之后的扩散型应用
List<String> parentIds = (List<String>) result.get(Constants.PARENTID);
parentIds.add(dataId);
result.put(Constants.PARENTID, parentIds);
}
}
//新增计费
Map<String, Object> appResult = (Map<String, Object>) result.get(Constants.REQUEST_RESULT);
if(appResult !=null && !appResult.isEmpty()){
if(appResult.containsKey(Constants.STATUS)){
//应用处理成功才计费
if((int)appResult.get(Constants.STATUS) == 1){
//指定应用进行计费
if(billingConfig.getApps().containsKey(result.get(Constants.MODULE_ID).toString())){
log.info("应用：{}，进行计费。",result.get(Constants.MODULE_ID));
Billing(result);
}
}
}
}
//调试模型不考虑数据开关
if(result.containsKey(Constants.TRACE) && (boolean)result.get(Constants.TRACE)){
//数据持久化到es---异步写入---再考虑
ConfigCache.putQueue(ConfigCache.saveDataQueue, result);
}else{
//根据存储标识判断数据是否持久化 历史流程不包含这个标识FLAG_DATA，默认写入、含标识并且为1的是迭代后的版本 写入
if(!result.containsKey(Constants.FLAG_DATA) || (Integer)result.get(Constants.FLAG_DATA) == 1){
//数据持久化到es---异步写入---再考虑
ConfigCache.putQueue(ConfigCache.saveDataQueue, result);
}
}
log.info("调回信息---场景id：{}，节点id{}",result.get(Constants.SCENES_ID),result.get(Constants.APP_ID));
List<Map<String, Object>> nextAppIds = (List<Map<String, Object>>) result.get(Constants.NEXT_APP_ID);
//二次推送获取完整data ---覆盖标识cover
if(result.containsKey(Constants.COVER)){
log.info("结果修改重新进行流程处理。节点id:{}",result.get(Constants.APP_ID));
//先获取当前数据data
Map<String, Object> data = (Map<String, Object>)result.get(Constants.DATA);
if(data == null){
data = new HashMap<String, Object>(16);
}
try {
//组装data
if(!result.containsKey(Constants.DATAPROCESSID)){
log.error("结果修改缺少必要参数：{}",Constants.DATAPROCESSID);
}
//扩散型数据修改
if(result.containsKey(Constants.PARENTID) && result.get(Constants.PARENTID) != null){
Map<String, Object> dataIds =new HashMap<String, Object>(16);
List<String> parentIds = (List<String>) result.get(Constants.PARENTID);
//没有父节点的数据源
String requestBody = String.format("{\"query\":{\"bool\":{\"must\":[{\"match\":{\"dataProcessId\":\"%s\"}}],\"must_not\":[{\"exists\":{\"field\":\"%s\"}}],\"should\":[]}}}",(String)result.get(Constants.DATAPROCESSID),Constants.PARENTID);
Map<String, Object> dataProcessResult = JSONObject.parseObject(ElasticsearchUtil.getDocument(nodes,username,password,indexName,requestBody));
Map<String, Object> hits = (Map<String, Object>) dataProcessResult.get(Constants.HITS);
List<Map<String, Object>> hitList = (List<Map<String, Object>>) hits.get(Constants.HITS);
if(hitList != null && hitList.size() > 0){
for (Map<String, Object> map : hitList) {
Map<String, Object> esSource = (Map<String, Object>) map.get(Constants.ES_SOURCE);
if(!data.containsKey(esSource.get(Constants.APPCODE))){
String resultStr = (String) esSource.get(Constants.REQUEST_RESULT);
data.put((String)esSource.get(Constants.APPCODE), resultStr);
//获取二次推送的节点的dataid
dataIds.put((String)esSource.get(Constants.APPCODE),esSource.get(Constants.DATAID));
}
}
result.put(Constants.DATAIDS, dataIds);
}
log.info("~~~~~~~~非扩散型应用数据获取完毕：{}",JSONObject.toJSONString(dataIds));
//有父节点的数据源
dataProcessResult = JSONObject.parseObject(ElasticsearchUtil.getDocumentBySingleCondition(Constants.PARENTID,parentIds.get(0),nodes,username,password,indexName));
hits = (Map<String, Object>) dataProcessResult.get(Constants.HITS);
hitList = (List<Map<String, Object>>) hits.get(Constants.HITS);
if(hitList != null && hitList.size() > 0){
for (Map<String, Object> map : hitList) {
Map<String, Object> esSource = (Map<String, Object>) map.get(Constants.ES_SOURCE);
if(!data.containsKey(esSource.get(Constants.APPCODE))){
String resultStr = (String) esSource.get(Constants.REQUEST_RESULT);
data.put((String)esSource.get(Constants.APPCODE), resultStr);
//获取二次推送的节点的dataid
dataIds.put((String)esSource.get(Constants.APPCODE),esSource.get(Constants.DATAID));
}
}
result.put(Constants.DATAIDS, dataIds);
}
log.info("`````扩散型应用数据获取完毕：{}",JSONObject.toJSONString(dataIds));
}else{
Map<String, Object> dataProcessResult = JSONObject.parseObject(ElasticsearchUtil.getDocumentBySingleCondition(Constants.DATAPROCESSID,(String)result.get(Constants.DATAPROCESSID),nodes,username,password,indexName));
Map<String, Object> hits = (Map<String, Object>) dataProcessResult.get(Constants.HITS);
List<Map<String, Object>> hitList = (List<Map<String, Object>>) hits.get(Constants.HITS);
if(hitList != null && hitList.size() > 0){
Map<String, Object> dataIds =new HashMap<String, Object>(16);
for (Map<String, Object> map : hitList) {
Map<String, Object> esSource = (Map<String, Object>) map.get(Constants.ES_SOURCE);
if(!data.containsKey(esSource.get(Constants.APPCODE))){
String resultStr = (String) esSource.get(Constants.REQUEST_RESULT);
data.put((String)esSource.get(Constants.APPCODE), resultStr);
//获取二次推送的节点的dataid
dataIds.put((String)esSource.get(Constants.APPCODE),esSource.get(Constants.DATAID));
}
}
result.put(Constants.DATAIDS, dataIds);
}
}
//data赋值
result.put(Constants.DATA, data);
} catch (Exception e) {
log.error("获取主体数据异常：",e);
}
}
for (Map<String, Object> map : nextAppIds) {
//获取下一个任务节点id
String endId = (String) map.get(Constants.END_ID);
//当前节点为空或与结束节点一致则流程流转完毕9
if(endId == null || endId .equals(result.get(Constants.APP_CODE))){
//没有break，可能会有多个节点的扩散
log.info("到最后一个节点了，不进行扩散---------------");
continue;
}
//获取场景id
Integer scenesId = (Integer) result.get(Constants.SCENES_ID);
//获取下一节点任务
log.info("nextTask redis key is:{}",scenesId+Constants.UNDERLINE+endId);
Map<String, Object> nextTask = JSONObject.parseObject(stringRedisTemplate.opsForValue().get(scenesId+Constants.UNDERLINE+endId));
//结果组装
bulidNextTask(result,nextTask);
}
} catch (Throwable e) {
log.error("未知错误---",e);
}
}
}
/**
* 下个任务参数组装
* @param result
* @param nextTask
*/
private void bulidNextTask(Map<String, Object> result,Map<String, Object> nextTask){
Map<String, Object> newTask = new HashMap<String, Object>(32);
for (Entry<String, Object> entry:nextTask.entrySet()) {
newTask.put(entry.getKey(), entry.getValue());
}
try {
//补充data
newTask.put(Constants.DATA, result.get(Constants.DATA));
//二次推送标识补充， 先判断
if(result.containsKey(Constants.COVER)){
newTask.put(Constants.COVER, result.get(Constants.COVER));
Map<String, Object> dataIds = (Map<String, Object>) result.get(Constants.DATAIDS);
newTask.put(Constants.DATAID, dataIds.get(newTask.get(Constants.APP_CODE)));
}
//获取当前节点请求返回结果
Map<String, Object> requestResult = (Map<String, Object>) result.get(Constants.REQUEST_RESULT);
Map<String, Object> nextData = null;
if(newTask.get(Constants.DATA) == null || newTask.get(Constants.DATA).equals(Constants.EMPTY)){
nextData = new HashMap<String, Object>(16);
}else{
nextData = (Map<String, Object>) newTask.get(Constants.DATA);
}
//数据源类型应用 ------ 最后数据 标识组装
if(requestResult.containsKey(Constants.ISLAST)){
nextData.put(Constants.ISLAST, requestResult.get(Constants.ISLAST));
}
//数据标识传递，有就传递，没有不生成-####-该调度不针对数据处理，基于项目处理，数据id根据业务需求所增
if(result.containsKey(Constants.BUSINESSKEY)){
newTask.put(Constants.BUSINESSKEY, result.get(Constants.BUSINESSKEY));
nextData.put(Constants.BUSINESSKEY, result.get(Constants.BUSINESSKEY));
}
String retStr = (String) requestResult.get(Constants.REQUEST_RESULT_RESULTS);
//data赋值此节点结果
nextData.put((String) result.get(Constants.APP_CODE), retStr);
//多分支场景缓存到redis 结果集
if(newTask.containsKey(Constants.MULTI_BRANCH) && (Integer)newTask.get(Constants.MULTI_BRANCH) == 1){
String dataKey = null;
if(result.containsKey(Constants.BUSINESSKEY)){
String businessKey = (String) result.get(Constants.BUSINESSKEY);
dataKey = businessKey+Constants.UNDERLINE+result.get(Constants.SCENES_ID)+Constants.UNDERLINE+result.get(Constants.SCENES_ID);
}else{
dataKey = result.get(Constants.SCENES_ID)+Constants.UNDERLINE+result.get(Constants.SCENES_ID);
}
dataCache(dataKey,(String)result.get(Constants.APP_CODE),retStr);
//all 结果汇总
Map<String, Object> nextOutput = (Map<String, Object>) nextTask.get(Constants.OUTPUT);
if(nextOutput.containsKey(Constants.DATA)){
Map<String, String> redisData = getDataCache(dataKey);
nextData.putAll(redisData);
log.info("All api result merge");
deleteDataCache(dataKey);
log.info("delete data cache");
}
}
newTask.put(Constants.DATA, nextData);
//数据流程主题id
newTask.put(Constants.DATAPROCESSID, result.get(Constants.DATAPROCESSID));
//扩散型父节点id ----聚会类应用不需要此字段
if(result.containsKey(Constants.PARENTID)){
if(!result.containsKey(Constants.POLYMERIZATION) || !(boolean)result.get(Constants.POLYMERIZATION)){
newTask.put(Constants.PARENTID, result.get(Constants.PARENTID));
}
}
//newTask或string直接写入队列
ConfigCache.putQueue(ConfigCache.taskQueue, newTask);
} catch (Exception e) {
// TODO: handle exception
log.error("构建次级任务失败---",e);
}
}
/**
* data在redis缓存
* @param dataKey
* @param key
* @param value
*/
private void dataCache(String dataKey,String key,String value){
HashOperations<String, String, String> hashOps = stringRedisTemplate.opsForHash();
// Set individual fields and values
hashOps.put(dataKey, key, value);
}
/**
* 取出redis中的data
* @param dataKey
* @return
*/
private Map<String, String> getDataCache(String dataKey){
HashOperations<String, String, String> hashOps = stringRedisTemplate.opsForHash();
Map<String, String> map = hashOps.entries(dataKey);
return map;
}
/**
* 删除缓存
* @param dataKey
*/
private void deleteDataCache(String dataKey){
stringRedisTemplate.delete(dataKey);
}
/**
* 计费接口调用
* @param data
*/
private void Billing(Map<String, Object> data){
Map<String, Object> params = new HashMap<String, Object>(16);
params.put(Constants.PLANCODE, data.get(Constants.PLANCODE));
params.put(Constants.SCENESID, data.get(Constants.SCENES_ID));
params.put(Constants.NOTE, data.get(Constants.SCENES_NAME));
params.put(Constants.DATAID, data.get(Constants.DATAID));
Map<String, Object> input = (Map<String, Object>) data.get(Constants.INPUT);
StringBuffer module = new StringBuffer();
module.append(Constants.CDA_);
module.append(data.get(Constants.MODULE_ID));
if((int)data.get(Constants.MODULE_ID) == billingConfig.getAimodelType()){
//智能模型分多个子模型，区分处理
module.append("_");
module.append(input.get(Constants.MODELTYPE));
}
params.put(Constants.MODULE, module.toString());
log.info("接口地址：{}，接口参数：{}",billingConfig.getBillingUrl() + Constants.BILLINGSUFFIX,JSONObject.toJSONString(params));
String response = DownLoadUtil.doPost(billingConfig.getBillingUrl() + Constants.BILLINGSUFFIX, JSONObject.toJSONString(params));
log.info("计费接口响应结果：{}",response);
}
}

48
src/main/java/com/bfd/analyze/process/ResultSendQueue.java
View File

@ -0,0 +1,48 @@
package com.bfd.analyze.process;
import com.alibaba.fastjson.JSONObject;
import com.bfd.analyze.cache.ConfigCache;
import com.bfd.analyze.entity.Constants;
import com.bfd.analyze.utils.PauseTool;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;
import java.util.Map;
/**
* 接口响应结果操作类
* @author jian.mao
* @date 2023年7月6日
* @description
*/
@Component
@Slf4j
public class ResultSendQueue {
@Autowired
PauseTool pauseTool;
/**
* kafka读取结果写入队列
* @param message
*/
@KafkaListener(topics = "#{kafkaConfig.getKafkaTopic()}")
public void consumeMessage(String message) {
// 处理接收到的消息逻辑
try {
Map<String, Object> result = JSONObject.parseObject(message);
Integer scense_id = (Integer) result.get(Constants.SCENES_ID);
Integer version = (Integer) result.get(Constants.VERSION);
//结果修改可不判断是否暂停
if (result.containsKey(Constants.COVER) || pauseTool.check(scense_id,version)) {
ConfigCache.ResultQueue.put(result);
}else {
log.info("暂停任务：{}",JSONObject.toJSONString(result));
}
} catch (Exception e) {
// TODO: handle exception
log.error("结果集json转换失败，result:{},\n",message,e);
}
}
}

57
src/main/java/com/bfd/analyze/process/SaveDataProcess.java
View File

@ -0,0 +1,57 @@
package com.bfd.analyze.process;
import com.alibaba.fastjson.JSONObject;
import com.bfd.analyze.cache.ConfigCache;
import com.bfd.analyze.entity.Constants;
import com.bfd.analyze.service.DataService;
import com.bfd.analyze.utils.PauseTool;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Map;
/**
* 数据持久化--es
* @author jian.mao
* @date 2023年7月7日
* @description
*/
@Component
@Slf4j
public class SaveDataProcess implements Runnable {
@Autowired
DataService dataService;
@Autowired
PauseTool pauseTool;
@Override
public void run() {
while(ConfigCache.isStart){
try {
Map<String, Object> result = null;
try {
result = ConfigCache.saveDataQueue.take();
Integer scense_id = (Integer) result.get(Constants.SCENES_ID);
Integer version = (Integer) result.get(Constants.VERSION);
if(result.containsKey(Constants.COVER)){
//结果修改不进行暂停任务判断
log.info("结果修改入库---");
}else if (!pauseTool.check(scense_id,version)) {
log.info("暂停任务：{}", JSONObject.toJSONString(result));
continue;
}
} catch (InterruptedException e) {
log.error("获取保存数据失败---",e);
}
log.info("数据保存信息---场景id：{}，节点id{}",result.get(Constants.SCENES_ID),result.get(Constants.APP_ID));
dataService.saveData(result);
} catch (Throwable e) {
// TODO: handle exception
log.error("未知错误---",e);
}
}
}
}

248
src/main/java/com/bfd/analyze/process/TaskdispatchProcess.java
View File

@ -0,0 +1,248 @@
package com.bfd.analyze.process;
import cn.hutool.core.lang.UUID;
import com.alibaba.fastjson.JSONObject;
import com.bfd.analyze.cache.ConfigCache;
import com.bfd.analyze.entity.AnalystData;
import com.bfd.analyze.entity.Constants;
import com.bfd.analyze.utils.DateUtil;
import com.bfd.analyze.utils.DownLoadUtil;
import com.bfd.analyze.utils.ElasticsearchUtil;
import com.bfd.analyze.utils.EncryptionUtil;
import com.bfd.analyze.utils.FileUtil;
import com.bfd.analyze.utils.PauseTool;
import com.google.gson.Gson;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
/**
* 任务调出线程
* @author jian.mao
* @date 2023年7月6日
* @description
*/
@Component
@Slf4j
public class TaskdispatchProcess implements Runnable{
@Value("${task.cycle-intv}")
private String intv;
@Autowired
PauseTool pauseTool;
@Value("${elasticsearch.dispatch.index-name}")
private String indexName;
@Value("${elasticsearch.cluster-nodes}")
private String nodes;
@Value("${elasticsearch.username}")
private String username;
@Value("${elasticsearch.password}")
private String password;
@Value("${task.error-result-path}")
private String path;
@Override
public void run() {
Gson gson = new Gson();
while(ConfigCache.isStart){
try {
Map<String, Object> task = null;
try {
task = ConfigCache.taskQueue.take();
Integer scense_id = (Integer) task.get(Constants.SCENES_ID);
Integer version = (Integer) task.get(Constants.VERSION);
if(task.containsKey(Constants.COVER)){
log.info("结果修改再次调出---");
}else if (!pauseTool.check(scense_id,version)) {
log.info("暂停任务：{}",JSONObject.toJSONString(task));
continue;
}else{
//数据调出写入唯一标识
String dataId = UUID.randomUUID().toString();
task.put(Constants.DATAID, dataId);
//放入调出id --写入调回数据里的
task.put(Constants.DISPATCHID, dataId);
}
} catch (InterruptedException e) {
log.error("获取调出任务失败---",e);
}
// log.info("调出任务：{}",JSONObject.toJSONString(task));
//调用接口 ---还是需要区分module，过滤器要在调度本身
String address = (String) task.get(Constants.ADDRESS);
//模型名称
String module = (String) task.get(Constants.MODULE);
//调出写入es -结果修改2次推送的，不计入统计
if(!task.containsKey(Constants.COVER)){
//调试模型不考虑数据开关
if(task.containsKey(Constants.TRACE) && (boolean)task.get(Constants.TRACE)){
saveData(task);
}else{
//根据存储标识判断数据是否持久化 历史流程不包含这个标识FLAG_DATA，默认写入、含标识并且为1的是迭代后的版本 写入
if(!task.containsKey(Constants.FLAG_DATA) || (Integer)task.get(Constants.FLAG_DATA) == 1){
saveData(task);
}
}
}
String content = DownLoadUtil.doPost(address, gson.toJson(task));
log.info("调出任务，scenes_id：{}，app_id：{}，module：{}",task.get(Constants.SCENES_ID),task.get(Constants.APP_ID),module);
if(content.contains(Constants.REQUEST_ERROR_MESSAGE)){
log.error("接口请求异常，scenes_id：{}，app_id：{}，module：{}，address：{}，errormessage：{}",task.get(Constants.SCENES_ID),task.get(Constants.APP_ID),module,address,content);
//删除调出索引的数据
ElasticsearchUtil.deleteDataById((String)task.get(Constants.DATAID), nodes, username, password, indexName);
//错误任务-处理
ConfigCache.errorTaskQueue.push(task);
}else{
log.info("接口返回值：{}",content);
}
//针对周期性app处理
// if((Integer)task.get(Constants.SCHEDULING_TYPE)==1){
// task.put(Constants.NEXT_SCHEDULING_TIME,System.currentTimeMillis()+intv);
// //周期任务
// ConfigCache.cycleTaskWaitQueue.put(task);
// }
} catch (Throwable e) {
log.error("未知错误----",e);
}
}
}
public void saveData(Map<String, Object> data) {
// TODO Auto-generated method stub
try {
AnalystData analystData = new AnalystData();
analystData.setAppId((Integer) data.get(Constants.APP_ID));
analystData.setAppCode((String) data.get(Constants.APP_CODE));
analystData.setCreated(DateUtil.getDateTime());
analystData.setLastEdit(DateUtil.getDateTime());
analystData.setTransferId((Integer) data.get(Constants.TRANSFER_ID));
analystData.setAppName((String) data.get(Constants.APP_NAME));
analystData.setAppDescribe((String) data.get(Constants.DESCRIBE));
analystData.setModule((String) data.get(Constants.MODULE));
analystData.setNextAppId((List<Map<String, Object>>) data.get(Constants.NEXT_APP_ID));
analystData.setStartTag((Boolean) data.get(Constants.START_TAG));
analystData.setScenesId((Integer) data.get(Constants.SCENES_ID));
analystData.setBusinessKey(data.containsKey(Constants.BUSINESSKEY)?data.get(Constants.BUSINESSKEY).toString():Constants.EMPTY);
if (data.containsKey(Constants.POLYMERIZATION) && (boolean)data.get(Constants.POLYMERIZATION)){
//多合一类应用调出数据只保留一条
String dataId = data.get(Constants.APP_CODE).toString();
analystData.setDataId(dataId);
data.put(Constants.DATAID, dataId);
//放入调出id --写入调回数据里的
data.put(Constants.DISPATCHID, dataId);
log.info("多合一类应用调出：{}",dataId);
}else{
analystData.setDataId(data.containsKey(Constants.DATAID)?data.get(Constants.DATAID).toString():UUID.randomUUID().toString());
}
analystData.setDel(0);
if(data.containsKey(Constants.DEFAULTCOMPLETION) && (boolean)data.get(Constants.DEFAULTCOMPLETION)){
//有些应用不需要调回去同步完成状态，比如过滤器，直接完成
analystData.setStatus(1);
}else{
analystData.setStatus(0);
}
//trace，0正式数据，1是测试数据
if(data.containsKey(Constants.TRACE) && (boolean)data.get(Constants.TRACE)){
analystData.setTrace(1);
}else{
analystData.setTrace(0);
}
//es数据添加version 用于删除判断
Integer version = (Integer) data.get(Constants.VERSION);
analystData.setVersion(version);
boolean status = save(analystData);
if(!status){
FileUtil.writeFile(path, JSONObject.toJSONString(data));
}
} catch (Exception e) {
log.error("调出任务转换es实体失败----",e);
}
// }
}
/**
* 节点数据持久化
* @param analystData
* @return
*/
private boolean save(AnalystData analystData){
//只写入一个节点
String node = nodes.split(",")[0];
// 创建凭据提供者
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
// 创建一个 HttpClient，设置凭据提供者
CloseableHttpClient httpClient = null;
try {
// 创建一个 HTTP POST 请求用于写入数据到索引
if(username != null && !username.trim().equals(Constants.EMPTY)){
// 创建凭据提供者
httpClient = HttpClients.custom()
.setDefaultCredentialsProvider(credentialsProvider)
.build();
}else{
httpClient = HttpClients.custom().build();
}
StringBuffer host = new StringBuffer("http://");
host.append(node);
host.append("/");
host.append(indexName);
host.append("/_doc/");
// Elasticsearch主机、索引名称和文档ID
host.append(analystData.getDataId());
HttpPost httpPost = new HttpPost(host.toString());
// 设置请求体，包含要写入的文档数据
StringEntity entity = new StringEntity(JSONObject.toJSONString(analystData), ContentType.APPLICATION_JSON);
httpPost.setEntity(entity);
// 发送请求并获取响应
HttpResponse response = httpClient.execute(httpPost);
// 处理响应
int statusCode = response.getStatusLine().getStatusCode();
String responseBody = EntityUtils.toString(response.getEntity());
int code = 201;
int updateCode = 200;
if (statusCode == code) {
log.info("调出任务数据成功写入到索引：{}，文档ID：{}，appid：{}",indexName,analystData.getDataId(),analystData.getAppId());
return true;
}else if(statusCode == updateCode){
log.info("调出任务数据成功更新到索引：{}，文档ID：{}，appid：{}",indexName,analystData.getDataId(),analystData.getAppId());
return true;
} else {
log.error("调出任务数据写入失败：{}，文档ID：{}，appid：{}，es响应内容：{}",indexName,analystData.getDataId(),analystData.getAppId(),responseBody);
return false;
}
} catch (Exception e) {
log.error("调出任务数据写入异常：",e);
return false;
}
}
}

18
src/main/java/com/bfd/analyze/service/DataService.java
View File

@ -0,0 +1,18 @@
package com.bfd.analyze.service;
import java.util.Map;
/**
* 数据处理业务层
* @author jian.mao
* @date 2023年7月11日
* @description
*/
public interface DataService {
/**
* 数据存储
*/
public void saveData(Map<String, Object> data);
}

23
src/main/java/com/bfd/analyze/service/DistributedLockService.java
View File

@ -0,0 +1,23 @@
package com.bfd.analyze.service;
/**
*
* @author jian.mao
* @date 2023年7月25日
* @description
*/
public interface DistributedLockService {
/**
* redis加锁
* @return
*/
public boolean acquireLock();
/**
* 释放锁
* redis
*/
public void releaseLock();
}

59
src/main/java/com/bfd/analyze/service/ProcessTriggerService.java
View File

@ -0,0 +1,59 @@
package com.bfd.analyze.service;
/**
* @PROJECT_NAME: analyst_assistant_schedule
* @DESCRIPTION:
* @AUTHOR: jian.mao
* @DATE: 2023/10/20 17:28
*/
public interface ProcessTriggerService {
/**
* @param param
* @return
*/
public String start(String param);
/**
* @param param
* @return
*/
public String stop(String param);
/**
* @param dataJson
* @return
*/
public String active(String param);
/**
* @param dataJson
* @return
*/
public String pause(String dataJson);
/**
* @param dataJson
* @return
*/
public String delete(String dataJson);
/**
* @param dataJson
* @return
*/
public String retry(String dataJson);
/**
* @param dataJson
* @return
*/
public String scenesDelete(String dataJson);
}

240
src/main/java/com/bfd/analyze/service/impl/DataServiceImpl.java
View File

@ -0,0 +1,240 @@
package com.bfd.analyze.service.impl;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSONObject;
import com.bfd.analyze.config.BillingConfig;
import com.bfd.analyze.entity.AnalystData;
import com.bfd.analyze.entity.Constants;
import com.bfd.analyze.service.DataService;
import com.bfd.analyze.utils.DateUtil;
import com.bfd.analyze.utils.DownLoadUtil;
import com.bfd.analyze.utils.ElasticsearchUtil;
import com.bfd.analyze.utils.FileUtil;
/**
* 数据处理业务层接口实现类
* @author jian.mao
* @date 2023年7月11日
* @description
*/
@Service
@Slf4j
public class DataServiceImpl implements DataService {
@Value("${elasticsearch.index-name}")
private String indexName;
@Value("${elasticsearch.dispatch.index-name}")
private String dispatchIndexName;
@Value("${elasticsearch.cluster-nodes}")
private String nodes;
@Value("${elasticsearch.username}")
private String username;
@Value("${elasticsearch.password}")
private String password;
@Value("${task.error-result-path}")
private String path;
@SuppressWarnings("unchecked")
@Override
public void saveData(Map<String, Object> data) {
// TODO Auto-generated method stub
Map<String, Object> result = (Map<String, Object>) data.get(Constants.REQUEST_RESULT);
// log.info("要保存的数据----{}",JSONObject.toJSONString(data));
if(result == null || result.isEmpty()){
log.error("返回数据无响应结果-----场景id：{}，节点id{}",data.get(Constants.SCENES_ID),data.get(Constants.APP_ID));
return ;
}
//获取当前节点请求返回结果
String retStr = (String) result.get(Constants.REQUEST_RESULT_RESULTS);
if(retStr == null || retStr.equals(Constants.EMPTY)){
log.error("响应结果无数据体-----场景id：{}，节点id{}",data.get(Constants.SCENES_ID),data.get(Constants.APP_ID));
// return ;
}
//过滤器内容干掉
// if(data.get(Constants.BLUEPRINT_NAME).equals(Constants.FILTER_ZH)){
// return;
// }
// String dataType = JsonUtil.checkJsonType(retStr);
//如果是list，遍历分多条任务发送
// if(dataType.equals(Constants.LIST_TYPE)){
// List results = JSONObject.parseArray(retStr);
// for (Object object : results) {
// try {
// AnalystData analystData = new AnalystData();
// analystData.setAppId((Integer) data.get(Constants.APP_ID));
// analystData.setAppCode((String) data.get(Constants.APP_CODE));
// analystData.setCreated(DateUtil.getDateTime());
// analystData.setLastEdit(DateUtil.getDateTime());
// analystData.setTransferId((Integer) data.get(Constants.TRANSFER_ID));
// analystData.setAppName((String) data.get(Constants.APP_NAME));
// analystData.setAppDescribe((String) data.get(Constants.DESCRIBE));
// analystData.setModule((String) data.get(Constants.MODULE));
// analystData.setResult(JSONObject.toJSONString(object));
// analystData.setNextAppId((List<Map<String, Object>>) data.get(Constants.NEXT_APP_ID));
// analystData.setStartTag((Boolean) data.get(Constants.START_TAG));
// analystData.setScenesId((Integer) data.get(Constants.SCENES_ID));
// analystData.setBusinessKey(data.containsKey(Constants.BUSINESSKEY)?data.get(Constants.BUSINESSKEY).toString():Constants.EMPTY);
// analystData.setDataId((String)data.get(Constants.DATAID));
// analystData.setDataProcessId((String)data.get(Constants.DATAPROCESSID));
// boolean status = save(analystData);
// if(!status){
// FileUtil.writeFile(path, JSONObject.toJSONString(data));
// }
// } catch (Exception e) {
// log.error("数据响应体转换es实体失败----",e);
// }
// }
// }else{
//map或string直接写入队列
try {
AnalystData analystData = new AnalystData();
analystData.setAppId((Integer) data.get(Constants.APP_ID));
analystData.setAppCode((String) data.get(Constants.APP_CODE));
analystData.setCreated(String.valueOf(System.currentTimeMillis()));
analystData.setLastEdit(String.valueOf(System.currentTimeMillis()));
analystData.setTransferId((Integer) data.get(Constants.TRANSFER_ID));
analystData.setAppName((String) data.get(Constants.APP_NAME));
analystData.setAppDescribe((String) data.get(Constants.DESCRIBE));
analystData.setModule((String) data.get(Constants.MODULE));
analystData.setResult(retStr==null?Constants.EMPTY:retStr);
analystData.setNextAppId((List<Map<String, Object>>) data.get(Constants.NEXT_APP_ID));
analystData.setStartTag((Boolean) data.get(Constants.START_TAG));
analystData.setScenesId((Integer) data.get(Constants.SCENES_ID));
analystData.setBusinessKey(data.containsKey(Constants.BUSINESSKEY)?data.get(Constants.BUSINESSKEY).toString():Constants.EMPTY);
analystData.setDataId((String)data.get(Constants.DATAID));
analystData.setDataProcessId((String)data.get(Constants.DATAPROCESSID));
analystData.setDispatchId(data.containsKey(Constants.DISPATCHID)?data.get(Constants.DISPATCHID).toString():Constants.EMPTY);
analystData.setDel(0);
if(result.containsKey(Constants.SEARCHFIELDS)){
analystData.setVariable((List<Map<String, Object>>) result.get(Constants.SEARCHFIELDS));
}
if(result.containsKey(Constants.STATUS)){
analystData.setStatus((int)result.get(Constants.STATUS));
analystData.setMessage((String) result.get(Constants.MESSAGE));
}
//有parentId就写入
if(data.containsKey(Constants.PARENTID)){
if(!data.containsKey(Constants.POLYMERIZATION) || !(boolean)data.get(Constants.POLYMERIZATION)){
analystData.setParentId((List<String>)data.get(Constants.PARENTID));
}
}
//trace，0正式数据，1是测试数据
if(data.containsKey(Constants.TRACE) && (boolean)data.get(Constants.TRACE)){
analystData.setTrace(1);
}else{
analystData.setTrace(0);
}
//es数据添加version 用于删除判断
Integer version = (Integer) data.get(Constants.VERSION);
analystData.setVersion(version);
boolean status = save(analystData);
if(!status){
FileUtil.writeFile(path, JSONObject.toJSONString(data));
}
//更新调出完成状态
try {
JSONObject results = JSONObject.parseObject(retStr);
if(results.containsKey(Constants.ISLAST)){
//数据已调回，通知调出索引
if(data.containsKey(Constants.DISPATCHID)){
String updateResultStr = ElasticsearchUtil.updateFieldById(dispatchIndexName, data.get(Constants.DISPATCHID).toString(),username, password, Constants.STATUS, 1, nodes);
log.info("数据已调回，更新调出完成状态：{}",updateResultStr);
}
}
} catch (Exception e) {
// TODO: handle exception
log.error("更新调出完成状态失败：{}",JSONObject.toJSONString(data));
}
} catch (Exception e) {
log.error("数据响应体转换es实体失败----",e);
}
// }
}
/**
* 节点数据持久化
* @param analystData
* @return
*/
private boolean save(AnalystData analystData){
//只写入一个节点
String node = nodes.split(",")[0];
// 创建凭据提供者
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
// 创建一个 HttpClient，设置凭据提供者
CloseableHttpClient httpClient = null;
try {
// 创建一个 HTTP POST 请求用于写入数据到索引
if(username != null && !username.trim().equals(Constants.EMPTY)){
// 创建凭据提供者
httpClient = HttpClients.custom()
.setDefaultCredentialsProvider(credentialsProvider)
.build();
}else{
httpClient = HttpClients.custom().build();
}
StringBuffer host = new StringBuffer("http://");
host.append(node);
host.append("/");
host.append(indexName);
host.append("/_doc/");
// Elasticsearch主机、索引名称和文档ID
host.append(analystData.getDataId());
HttpPost httpPost = new HttpPost(host.toString());
// 设置请求体，包含要写入的文档数据
StringEntity entity = new StringEntity(JSONObject.toJSONString(analystData), ContentType.APPLICATION_JSON);
httpPost.setEntity(entity);
// 发送请求并获取响应
HttpResponse response = httpClient.execute(httpPost);
// 处理响应
int statusCode = response.getStatusLine().getStatusCode();
String responseBody = EntityUtils.toString(response.getEntity());
int code = 201;
int updateCode = 200;
if (statusCode == code) {
log.info("数据成功写入到索引：{}，文档ID：{}，appid：{}",indexName,analystData.getDataId(),analystData.getAppId());
return true;
}else if(statusCode == updateCode){
log.info("数据成功更新到索引：{}，文档ID：{}，appid：{}",indexName,analystData.getDataId(),analystData.getAppId());
return true;
} else {
log.error("数据写入失败：{}，文档ID：{}，appid：{}，es响应内容：{}",indexName,analystData.getDataId(),analystData.getAppId(),responseBody);
return false;
}
} catch (Exception e) {
log.error("数据写入异常：",e);
return false;
}
}
}

40
src/main/java/com/bfd/analyze/service/impl/DistributedLockServiceImpl.java
View File

@ -0,0 +1,40 @@
package com.bfd.analyze.service.impl;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import com.bfd.analyze.entity.Constants;
import com.bfd.analyze.service.DistributedLockService;
/**
* @author jian.mao
* @date 2023年7月31日
* @description
*/
@Service
@Slf4j
public class DistributedLockServiceImpl implements DistributedLockService {
@Autowired
private StringRedisTemplate redisTemplate;
@Override
public boolean acquireLock() {
Boolean isLocked = redisTemplate.opsForValue().setIfAbsent(Constants.LOCK_KEY,
"locked", Constants.LOCK_EXPIRE_TIME, TimeUnit.MILLISECONDS);
return isLocked != null && isLocked;
}
@Override
public void releaseLock() {
// TODO Auto-generated method stub
redisTemplate.delete(Constants.LOCK_KEY);
}
}

637
src/main/java/com/bfd/analyze/service/impl/ProcessTriggerServiceImpl.java
View File

@ -0,0 +1,637 @@
package com.bfd.analyze.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.bfd.analyze.cache.ConfigCache;
import com.bfd.analyze.entity.Constants;
import com.bfd.analyze.mapper.AppsMapper;
import com.bfd.analyze.mapper.ModulesMapper;
import com.bfd.analyze.mapper.RelationsMapper;
import com.bfd.analyze.mapper.ScenesMapper;
import com.bfd.analyze.model.AppsEntity;
import com.bfd.analyze.model.ModulesEntity;
import com.bfd.analyze.model.RelationsEntity;
import com.bfd.analyze.model.ScenesEntity;
import com.bfd.analyze.service.ProcessTriggerService;
import com.bfd.analyze.utils.ElasticsearchUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;
/**
* @PROJECT_NAME: analyst_assistant_schedule
* @DESCRIPTION:
* @AUTHOR: jian.mao
* @DATE: 2023/10/20 17:37
*/
@Slf4j
@Service
public class ProcessTriggerServiceImpl implements ProcessTriggerService {
@Resource
private AppsMapper appsMapper;
@Resource
private ModulesMapper modulesMapper;
@Resource
private RelationsMapper relationsMapper;
@Resource
private ScenesMapper scenesMapper;
@Autowired
private StringRedisTemplate stringRedisTemplate;
@Autowired
private CuratorFramework curatorFramework;
@Value("${zookeeper.publish-node}")
private String publishNode;
private InterProcessMutex lock;
@Value("${elasticsearch.index-name}")
private String indexName;
@Value("${elasticsearch.dispatch.index-name}")
private String dispatchIndexName;
@Value("${elasticsearch.cluster-nodes}")
private String nodes;
@Value("${elasticsearch.username}")
private String username;
@Value("${elasticsearch.password}")
private String password;
@PostConstruct
public void init() {
lock = new InterProcessMutex(curatorFramework, publishNode);
}
@Override
public String start(String param) {
Map<String, Object> response = new HashMap<String, Object>(16);
Map<String, Object> zkResult = new HashMap<String, Object>(16);
int code = 200;
String message = "success";
int version = 0;
try {
Map<String, Object> params = JSONObject.parseObject(param);
Integer scenesId = Integer.valueOf(params.get(Constants.SCENES_ID).toString());
//先清除缓存
log.info("清除场景：{} 的流程缓存", scenesId);
deleteKeysWithPrefix(scenesId + Constants.UNDERLINE);
String scenes_version = stringRedisTemplate.opsForValue().get(Constants.VERSION + Constants.UNDERLINE + scenesId);
log.info("清除场景：{} ,verison不等于 {} 的历史数据", scenesId,scenes_version);
//异步删除
Thread thread = new Thread(() -> {
log.info("流程开启--->异步删除历史数据----start");
//调回数据删除
ElasticsearchUtil.deleteDataBycondition(Constants.SCENESID, scenesId, nodes, username, password, indexName,scenes_version);
//调出数据删除
ElasticsearchUtil.deleteDataBycondition(Constants.SCENESID, scenesId, nodes, username, password, dispatchIndexName,scenes_version);
log.info("流程开启--->异步删除历史数据----end");
});
// 启动线程
thread.start();
// 尝试获取分布式锁，启动通知应用
if (lock.acquire(Constants.LOCKTIME, TimeUnit.SECONDS)) {
try {
log.info("获取锁成功------------{}", param);
//zk 通知 启动的场景 和 版本
zkResult.put("scenes_id", scenesId);
zkResult.put("operation", Constants.START);
//从redis尝试获取version,根据是否有version,判断是否是重新启动的流程
if (scenes_version != null) {
//重新启动的流程 拿到version
version = Integer.parseInt(scenes_version);
} else {
//场景第一次启动 将version存储在redis
stringRedisTemplate.opsForValue().set(Constants.VERSION + Constants.UNDERLINE + scenesId, String.valueOf(version));
//将 场景版本号 存储到内存中，便于去除暂停任务
}
zkResult.put("version", version);
String value = JSONObject.toJSONString(zkResult);
log.info("zk通知消息-----------{}",value);
// String newKey = scenesId + Constants.UNDERLINE + version;
// // 移除CACHE中所有以scenesId开头的key
// PauseTool.CACHE.keySet().removeIf(key -> key.startsWith(scenesId + Constants.UNDERLINE));
// PauseTool.CACHE.put(newKey, version);
//通知应用 将场景_版本号 写到缓存里
if (curatorFramework.checkExists().forPath(publishNode) == null) {
curatorFramework.create().creatingParentsIfNeeded().forPath(publishNode, value.getBytes());
} else {
// 如果节点存在，则更新节点数据
curatorFramework.setData().forPath(publishNode, value.getBytes());
}
message = "Node updated successfully";
} finally {
// 释放锁
lock.release();
}
//获取场景信息
ScenesEntity scenes = scenesMapper.getscenesById(scenesId);
//获取场景下的节点信息
List<AppsEntity> apps = appsMapper.getAppByscenesId(scenesId);
if (apps != null && apps.size() > 0) {
Map<String, Object> firstTask = new HashMap<String, Object>(16);
for (AppsEntity app : apps) {
//获取modules
ModulesEntity module = modulesMapper.getModuleByModuleId(app.getModuleId());
//获取relations
List<RelationsEntity> relations = relationsMapper.getRelationsByscenesId(app.getScenesId());
log.info("组装节点：{}，组装参数：{}，", JSONObject.toJSONString(app), JSONObject.toJSONString(params));
//构建任务
Map<String, Object> task = bulidTask(app, module, relations, params, scenes, version);
//任务发送
send(task, firstTask);
}
//首节点直接调出
if (!firstTask.isEmpty()) {
try {
//调试场景优先调出
if(params.containsKey(Constants.TRACE) && (boolean)params.get(Constants.TRACE)){
log.info("调式任务写入队列-----------");
ConfigCache.taskQueue.putFirst(firstTask);
}else{
ConfigCache.taskQueue.put(firstTask);
}
} catch (InterruptedException e) {
log.error("任务写入队列异常------");
}
}
} else {
//查询不到流程
code = 100001;
message = "查询不到流程";
}
} else {
code = 100001;
message = "Failed to acquire lock";
log.warn("获取锁失败-----------{}", param);
}
} catch (Exception e) {
code = 100002;
message = "流程节点组装异常";
log.error("异常信息，", e);
}
response.put(Constants.CODE, code);
response.put(Constants.MESSAGE, message);
response.put(Constants.VERSION, version);
return JSONObject.toJSONString(response);
}
private void send(Map<String, Object> task, Map<String, Object> firstTask) {
if ((Integer) task.get(Constants.TRANSFER_ID) > 0) {
log.info("次级任务写入缓存中等待：场景--{}，app节点--{}", task.get(Constants.SCENES_ID), task.get(Constants.APP_NAME));
log.info("次级节点任务：{}", JSONObject.toJSONString(task));
//redis缓存###key存储规则：蓝图scenesId_appId
stringRedisTemplate.opsForValue().set(task.get(Constants.SCENES_ID) + Constants.UNDERLINE + task.get(Constants.APP_CODE), JSONObject.toJSONString(task));
} else {
//判断是否主动调用的应用，配置流程后设置数据的应用 ，比如文件上传
Map<String, Object> input = (Map<String, Object>) task.get(Constants.INPUT);
if (input.containsKey(Constants.IS_ACTIVE) && (boolean) input.get(Constants.IS_ACTIVE)) {
log.info("该节点为主动触发应用：场景--{}，app节点--{}", task.get(Constants.SCENES_ID), task.get(Constants.APP_NAME));
} else {
for (Entry<String, Object> entry : task.entrySet()) {
firstTask.put(entry.getKey(), entry.getValue());
}
}
}
}
/**
* 删除缓存key
*
* @param prefix
*/
public void deleteKeysWithPrefix(String prefix) {
Set<String> keys = stringRedisTemplate.keys(prefix + Constants.WILDCARD);
if (keys != null) {
keys.forEach(key -> stringRedisTemplate.delete(key));
}
}
private Map<String, Object> bulidTask(AppsEntity app, ModulesEntity module, List<RelationsEntity> relations, Map<String, Object> webParams, ScenesEntity scenes, int version) {
Map<String, Object> task = new HashMap<String, Object>(16);
//调试模型所需字段
if(webParams.containsKey(Constants.TRACE) && (boolean)webParams.get(Constants.TRACE)){
log.info("调试字段写入：{}",JSONObject.toJSONString(webParams));
task.put(Constants.TRACE, webParams.get(Constants.TRACE));
task.put(Constants.DATAPROCESSID, webParams.get(Constants.DATAPROCESSID));
}
//增加外部主键
task.put(Constants.SOURCE_DATA_ID, scenes.getSourceDataId());
//场景名称
task.put(Constants.SCENES_NAME, scenes.getScenesName());
//场景数据存储开关
task.put(Constants.FLAG_DATA, scenes.getFlagData());
//套餐标识
task.put(Constants.PLANCODE, webParams.get(Constants.PLANCODE));
//应用id
task.put(Constants.MODULE_ID, module.getId());
//增加操作用户id
task.put(Constants.CREATEUSERID, app.getCreateUserId());
//输出配置 ---需要解析
Map<String, Object> afterOutput = new HashMap<String, Object>(16);
String parseBeforOpStr = app.getOutput();
Map<String, Object> beforOutput = JSONObject.parseObject(parseBeforOpStr);
if (beforOutput.containsKey(Constants.FORM)) {
List<Map<String, Object>> form = (List<Map<String, Object>>) beforOutput.get(Constants.FORM);
for (Map<String, Object> map : form) {
String field = (String) map.get(Constants.FIELD);
afterOutput.put(field, field);
}
}
task.put(Constants.OUTPUT, afterOutput);
/***输入配置 = 固定配置+选配置***/
//固定配置
Map<String, Object> fixedParams = JSONObject.parseObject(module.getParams());
//选配置
Map<String, Object> input = JSONObject.parseObject(app.getInput());
//合并
fixedParams.putAll(input);
// input.putAll(fixedParams);
task.put(Constants.INPUT, fixedParams);
//接口地址
task.put(Constants.ADDRESS, fixedParams.get(Constants.ADDRESS));
//多合一应用标识
task.put(Constants.POLYMERIZATION, fixedParams.containsKey(Constants.POLYMERIZATION)?true:false);
//不设置完成状态标识
task.put(Constants.DEFAULTCOMPLETION, fixedParams.containsKey(Constants.DEFAULTCOMPLETION)?true:false);
//应用扩散标识
task.put(Constants.IS_DIFFUSION, fixedParams.containsKey(Constants.IS_DIFFUSION)?true:false);
//模块名称
task.put(Constants.MODULE, module.getName());
//流转内部节点id
task.put(Constants.TRANSFER_ID, app.getTransferId());
//场景id
task.put(Constants.SCENES_ID, app.getScenesId());
//节点名称
task.put(Constants.APP_NAME, app.getName());
//businessKey
// task.put(Constants.BUSINESSKEY,webParams.get(Constants.BUSINESSKEY));
task.put(Constants.BUSINESSKEY, UUID.randomUUID().toString());
//app调出模式
task.put(Constants.SCHEDULING_TYPE, app.getSchedulingType());
//ida
task.put(Constants.APP_ID, app.getId());
//节点描述
task.put(Constants.DESCRIBE, app.getDescribe());
//appCode
task.put(Constants.APP_CODE, app.getAppCode());
//开始节点start_tag
task.put(Constants.START_TAG, app.getTransferId() == 0 ? true : false);
//等待节点
task.put(Constants.WAIT_CONDITION, app.getWaitCondition());
//next_app
task.put(Constants.RELATIONS, relations);
List<Map<String, Object>> nextAppIds = new ArrayList<Map<String, Object>>();
for (RelationsEntity relation : relations) {
if (relation.getStartCode().equals(app.getAppCode())) {
Map<String, Object> nextApp = new HashMap<String, Object>(16);
nextApp.put(Constants.EDGE_ID, relation.getId());
nextApp.put(Constants.START_ID, relation.getStartCode());
nextApp.put(Constants.END_ID, relation.getEndCode());
nextAppIds.add(nextApp);
}
}
task.put(Constants.NEXT_APP_ID, nextAppIds);
task.put(Constants.VERSION, version);
return task;
}
@Override
public String stop(String param) {
Map<String, Object> response = new HashMap<String, Object>(16);
int code = 200;
String message = "success";
try {
// 尝试获取分布式锁
if (lock.acquire(Constants.LOCKTIME, TimeUnit.SECONDS)) {
try {
JSONObject data = JSONObject.parseObject(param);
String value = data.get(Constants.SCENES_ID).toString();
log.info("获取锁成功------------{}", param);
if (curatorFramework.checkExists().forPath(publishNode) == null) {
curatorFramework.create().creatingParentsIfNeeded().forPath(publishNode, value.getBytes());
} else {
// 如果节点存在，则更新节点数据
curatorFramework.setData().forPath(publishNode, value.getBytes());
}
message = "Node updated successfully";
} finally {
// 释放锁
lock.release();
}
} else {
code = 100001;
message = "Failed to acquire lock";
log.warn("获取锁失败-----------{}", param);
}
} catch (Exception e) {
log.error("未知异常：{}，{}", param, e);
code = 100002;
message = "Failed to update node: " + e.getMessage();
}
response.put(Constants.CODE, code);
response.put(Constants.MESSAGE, message);
return JSONObject.toJSONString(response);
}
@Override
public String active(String param) {
Map<String, Object> response = new HashMap<String, Object>(16);
int code = 200;
String message = "success";
try {
Map<String, Object> params = JSONObject.parseObject(param);
Integer scenesId = Integer.valueOf(params.get(Constants.SCENES_ID).toString());
String appCode = params.get(Constants.APP_CODE).toString();
//获取场景信息
ScenesEntity scenes = scenesMapper.getscenesById(scenesId);
//获取场景下的节点信息
AppsEntity app = appsMapper.getAppByAppCode(appCode);
if (app != null) {
//获取modules
ModulesEntity module = modulesMapper.getModuleByModuleId(app.getModuleId());
//获取relations
List<RelationsEntity> relations = relationsMapper.getRelationsByscenesId(app.getScenesId());
//设置输入配置
Map<String, Object> appInput = JSONObject.parseObject(app.getInput());
if(params.containsKey(Constants.INPUT)){
Map<String, Object> input = (Map<String, Object>) params.get(Constants.INPUT);
appInput.putAll(input);
}
app.setInput(JSONObject.toJSONString(appInput));
//构建任务
int version = 0;
String scenes_version = stringRedisTemplate.opsForValue().get(Constants.VERSION + Constants.UNDERLINE + scenesId);
if (scenes_version != null) {
//重新启动的流程 拿到version
version = Integer.parseInt(scenes_version);
}
Map<String, Object> task = bulidTask(app, module, relations, params, scenes, version);
//判断是否有数据体 ---系统参数
if (params.containsKey(Constants.DATA)) {
task.put(Constants.DATA, params.get(Constants.DATA));
}
//api接口结果传输
if(params.containsKey(Constants.REQUEST_RESULT)){
Object result = params.get(Constants.REQUEST_RESULT);
task.put(Constants.REQUEST_RESULT, result);
}
//任务发送
log.info("流程主动触发：{}，组装参数：{}，", JSONObject.toJSONString(app), JSONObject.toJSONString(params));
try {
ConfigCache.taskQueue.put(task);
} catch (InterruptedException e) {
log.error("任务写入队列异常------");
}
} else {
//查询不到流程
code = 100001;
message = "查询不到流程";
}
} catch (Exception e) {
code = 100002;
message = "流程节点组装异常";
log.error("异常信息，", e);
}
response.put(Constants.CODE, code);
response.put(Constants.MESSAGE, message);
return JSONObject.toJSONString(response);
}
@Override
public String pause(String param) {
Map<String, Object> response = new HashMap<String, Object>(16);
Map<String, Object> zkResult = new HashMap<String, Object>(16);
int code = 200;
String message = "success";
try {
// 尝试获取分布式锁
if (lock.acquire(Constants.LOCKTIME, TimeUnit.SECONDS)) {
try {
JSONObject data = JSONObject.parseObject(param);
//场景id
Integer scenes_id = Integer.valueOf(data.get(Constants.SCENES_ID).toString());
//zk 通知 停止的场景 和 版本
zkResult.put("scenes_id", scenes_id);
zkResult.put("operation", Constants.STOP);
String scenes_version = stringRedisTemplate.opsForValue().get(Constants.VERSION + Constants.UNDERLINE + scenes_id);
int version = scenes_version != null ? Integer.parseInt(scenes_version) + 1 : 0;
stringRedisTemplate.opsForValue().set(Constants.VERSION + Constants.UNDERLINE + scenes_id, String.valueOf(version));
zkResult.put("version", version);
// String newKey = scenes_id + Constants.SCENES_ID + version;
// 移除CACHE中所有以scenesId开头的key
// PauseTool.CACHE.keySet().removeIf(key -> key.startsWith(scenes_id + Constants.UNDERLINE));
// 将新的key放入CACHE
// PauseTool.CACHE.put(newKey, String.valueOf(version));
String value = JSONObject.toJSONString(zkResult);
log.info("获取锁成功------------{}", param);
log.info("zk通知消息-----------{}",value);
if (curatorFramework.checkExists().forPath(publishNode) == null) {
curatorFramework.create().creatingParentsIfNeeded().forPath(publishNode, value.getBytes());
} else {
// 如果节点存在，则更新节点数据
curatorFramework.setData().forPath(publishNode, value.getBytes());
}
message = "Node updated successfully";
} finally {
// 释放锁
lock.release();
}
} else {
code = 100001;
message = "Failed to acquire lock";
log.warn("获取锁失败-----------{}", param);
}
} catch (Exception e) {
log.error("未知异常：{}，{}", param, e);
code = 100002;
message = "Failed to update node: " + e.getMessage();
}
response.put(Constants.CODE, code);
response.put(Constants.MESSAGE, message);
return JSONObject.toJSONString(response);
}
@Override
public String delete(String dataJson) {
// TODO Auto-generated method stub
JSONObject parseObject = JSONObject.parseObject(dataJson);
String dataProcessId = parseObject.getString(Constants.DATAPROCESSID);
String appCode = parseObject.getString(Constants.APPCODE);
Map<String, Object> result = new HashMap<String, Object>(16);
int code = 1;
String message = "删除成功";
try {
if( parseObject.containsKey(Constants.PARENTID) && parseObject.get(Constants.PARENTID) != null ){
//删除调回索引中的数据
log.info("扩散型数据删除----");
List<String> parentIds = (List<String>) parseObject.get(Constants.PARENTID);
String deleteResponse = ElasticsearchUtil.deleteDataByconditionNoVersion(Constants.PARENTID + Constants.POINT_KEYWORD, parentIds.get(parentIds.size()-1), nodes, username, password, indexName,Constants.TERM);
JSONObject deleteObject = JSONObject.parseObject(deleteResponse);
if(deleteObject.containsKey(Constants.ERROR)){
message = "fial";
code = 40001;
}
//删除调出索引中数据
String searchResultStr = ElasticsearchUtil.getDocumentBySingleCondition(Constants.APPCODE, appCode, nodes, username, password, indexName);
JSONObject searchResult = JSONObject.parseObject(searchResultStr);
Map<String, Object> hits = (Map<String, Object>) searchResult.get(Constants.HITS);
List<Map<String, Object>> hitList = (List<Map<String, Object>>) hits.get(Constants.HITS);
if (hitList != null && hitList.size() <= 1) {
if(parseObject.containsKey(Constants.DISPATCHID)){
//获取调出id --调出数据删除不考虑version问题
String dispatchId = (String) parseObject.get(Constants.DISPATCHID);
ElasticsearchUtil.deleteDataBycondition(Constants.DATAID, dispatchId, nodes, username, password, dispatchIndexName,"-1");
}
}
}else{
//无扩散型应用数据删除
String deleteResponse = ElasticsearchUtil.deleteDataByconditionNoVersion(Constants.DATAPROCESSID, dataProcessId, nodes, username, password, indexName,Constants.MATCH);
JSONObject deleteObject = JSONObject.parseObject(deleteResponse);
if(deleteObject.containsKey(Constants.ERROR)){
message = "fial";
code = 40001;
}
if(parseObject.containsKey(Constants.DISPATCHID)){
//获取调出id --调出数据删除不考虑version问题
String dispatchId = (String) parseObject.get(Constants.DISPATCHID);
ElasticsearchUtil.deleteDataBycondition(Constants.DATAID, dispatchId, nodes, username, password, dispatchIndexName,"-1");
}
}
} catch (Exception e) {
log.error("删除失败-------{}", dataProcessId, e);
code = -1;
}
result.put(Constants.CODE, code);
result.put(Constants.MESSAGE, message);
return JSONObject.toJSONString(result);
}
@Override
public String retry(String dataJson) {
JSONObject parseObject = JSONObject.parseObject(dataJson);
String appCode = (String) parseObject.get(Constants.APP_CODE);
String conditionField = Constants.APPCODE;
Map<String, Object> result = new HashMap<String, Object>(16);
int code = 0;
String message = "未找到重试节点";
try {
String searchResultStr = ElasticsearchUtil.getDocumentBySingleCondition(conditionField, appCode, nodes, username, password, indexName);
JSONObject searchResult = JSONObject.parseObject(searchResultStr);
Map<String, Object> hits = (Map<String, Object>) searchResult.get(Constants.HITS);
List<Map<String, Object>> hitList = (List<Map<String, Object>>) hits.get(Constants.HITS);
if (hitList != null && hitList.size() > 0) {
for (Map<String, Object> map : hitList) {
Map<String, Object> esSource = (Map<String, Object>) map.get(Constants.ES_SOURCE);
Integer status = (Integer) esSource.get(Constants.STATUS);
//异常的重试
if (status != null && status == 2) {
//获取单条完整场景流程
Map<String, Object> data = new HashMap<String, Object>(16);
String dataProcessId = (String) esSource.get(Constants.DATAPROCESSID);
conditionField = Constants.DATAPROCESSID;
String singleResultStr = ElasticsearchUtil.getDocumentBySingleCondition(conditionField, dataProcessId, nodes, username, password, indexName);
JSONObject singleResult = JSONObject.parseObject(singleResultStr);
Map<String, Object> singleHits = (Map<String, Object>) singleResult.get(Constants.HITS);
List<Map<String, Object>> singleHitList = (List<Map<String, Object>>) singleHits.get(Constants.HITS);
for (Map<String, Object> item : singleHitList) {
Map<String, Object> singleEsSource = (Map<String, Object>) item.get(Constants.ES_SOURCE);
//结果集 data组装
if (!data.containsKey(singleEsSource.get(Constants.APPCODE))) {
String resultStr = (String) singleEsSource.get(Constants.REQUEST_RESULT);
data.put((String) singleEsSource.get(Constants.APPCODE), resultStr);
}
//组装结果集 --加到调出列中，从而触发当前节点的调用
if (appCode.equals(singleEsSource.get(Constants.APPCODE))) {
//构建任务
Map<String, Object> task = JSONObject.parseObject(stringRedisTemplate.opsForValue().get(singleEsSource.get(Constants.SCENESID) + Constants.UNDERLINE + appCode));
Map<String, Object> newTask = new HashMap<String, Object>(32);
for (Entry<String, Object> entry : task.entrySet()) {
newTask.put(entry.getKey(), entry.getValue());
}
newTask.put(Constants.DATAID, singleEsSource.get(Constants.DATAID));
newTask.put(Constants.DATAPROCESSID, singleEsSource.get(Constants.DATAPROCESSID));
newTask.put(Constants.COVER, true);
newTask.put(Constants.BUSINESSKEY, singleEsSource.get(Constants.BUSINESSKEY));
if(singleEsSource.containsKey(Constants.DISPATCHID)){
newTask.put(Constants.DISPATCHID, singleEsSource.get(Constants.DISPATCHID));
}
//data补充
newTask.put(Constants.DATA, data);
//newTask或string直接写入队列
ConfigCache.putQueue(ConfigCache.taskQueue, newTask);
code = 1;
message = "重试任务成功下发";
log.info("重试任务成功下发，app_code:{},dataId:{}", appCode, singleEsSource.get(Constants.DATAID));
}
}
}
}
}
} catch (IOException e) {
log.error("重试失败-------{}", dataJson, e);
code = -1;
}
result.put(Constants.CODE, code);
result.put(Constants.MESSAGE, message);
return JSONObject.toJSONString(result);
}
@Override
public String scenesDelete(String dataJson) {
Map<String, Object> response = new HashMap<String, Object>(16);
int code = 200;
String message = "success";
try {
Map<String, Object> params = JSONObject.parseObject(dataJson);
Integer scenesId = Integer.valueOf(params.get(Constants.SCENES_ID).toString());
//调回数据删除
String deleteResponse = ElasticsearchUtil.deleteDataByconditionNoVersion(Constants.SCENESID, scenesId, nodes, username, password, indexName,Constants.MATCH);
JSONObject parseObject = JSONObject.parseObject(deleteResponse);
if(parseObject.containsKey(Constants.ERROR)){
message = "fial";
code = 40001;
}else{
Map<String, Object> zkResult = new HashMap<String, Object>(16);
String scenesVersion = stringRedisTemplate.opsForValue().get(Constants.VERSION + Constants.UNDERLINE + scenesId);
zkResult.put("scenes_id", scenesId);
zkResult.put("operation", Constants.DELETE);
zkResult.put(Constants.VERSION, scenesVersion);
String value = JSONObject.toJSONString(zkResult);
//删除通知
curatorFramework.setData().forPath(publishNode, value.getBytes());
}
}catch (Exception e) {
// TODO: handle exception
log.error("删除未知异常：",e);
message = "fial";
code = 40002;
}
response.put(Constants.CODE, code);
response.put(Constants.MESSAGE, message);
return JSONObject.toJSONString(response);
}
}

79
src/main/java/com/bfd/analyze/test/CreateElasticsearchIndex.java
View File

@ -0,0 +1,79 @@
package com.bfd.analyze.test;
import java.util.Base64;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
/**
* @author jian.mao
* @date 2023年11月3日
* @description
*/
public class CreateElasticsearchIndex {
public static void main(String[] args) {
// Elasticsearch的主机和端口
HttpHost elasticsearchHost = new HttpHost("172.16.12.55", 9200);
// 用户名和密码
String username = "elastic";
String password = "bfd123";
// Elasticsearch索引名称
String indexName = "analyze_00001";
// 创建一个 HttpClient
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
// 创建一个 HTTP PUT 请求用于创建索引// Elasticsearch主机和索引名称
HttpPut httpPut = new HttpPut("http://172.16.12.55:9200/" + indexName);
// 设置基本认证
httpPut.addHeader("Authorization", "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes()));
// 设置请求体，定义索引的映射和设置
String indexDefinition = "{" +
" \"mappings\": {" +
" \"properties\": {" +
" \"created\": {\"type\": \"keyword\"}," +
" \"lastEdit\": {\"type\": \"keyword\"}," +
" \"appId\": {\"type\": \"integer\"}," +
" \"transferId\": {\"type\": \"integer\"}," +
" \"appName\": {\"type\": \"keyword\"}," +
" \"appCode\": {\"type\": \"keyword\"}," +
" \"appDescribe\": {\"type\": \"text\"}," +
" \"module\": {\"type\": \"keyword\"}," +
" \"result\": {\"type\": \"text\"}," +
" \"nextAppId\": {\"type\": \"nested\"}," +
" \"startTag\": {\"type\": \"boolean\"}," +
" \"scenesId\": {\"type\": \"integer\"}," +
" \"businessKey\": {\"type\": \"keyword\"}" +
" }" +
" }" +
"}";
httpPut.setEntity(new StringEntity(indexDefinition, ContentType.APPLICATION_JSON));
// 发送请求并获取响应
HttpResponse response = httpClient.execute(elasticsearchHost, httpPut);
// 处理响应
int statusCode = response.getStatusLine().getStatusCode();
String responseBody = EntityUtils.toString(response.getEntity());
int code = 200;
if (statusCode == code) {
System.out.println("索引创建成功");
} else {
System.out.println("索引创建失败：" + responseBody);
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

58
src/main/java/com/bfd/analyze/test/DeleteElasticsearchIndex.java
View File

@ -0,0 +1,58 @@
package com.bfd.analyze.test;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.impl.client.BasicCredentialsProvider;
/**
* @author jian.mao
* @date 2023年11月3日
* @description
*/
public class DeleteElasticsearchIndex {
public static void main(String[] args) {
// Elasticsearch的主机和端口
HttpHost elasticsearchHost = new HttpHost("172.16.12.55", 9200);
// Elasticsearch索引名称
String indexName = "analyze_00001";
// 用户名和密码
String username = "elastic";
String password = "bfd123";
// 创建凭据提供者
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
// 创建一个 HttpClient，设置凭据提供者
try (CloseableHttpClient httpClient = HttpClients.custom()
.setDefaultCredentialsProvider(credentialsProvider)
.build()) {
// 创建一个 HTTP DELETE 请求用于删除索引
HttpDelete httpDelete = new HttpDelete("http://172.16.12.55:9200/" + indexName);
// 发送请求并获取响应
HttpResponse response = httpClient.execute(elasticsearchHost, httpDelete);
// 处理响应
int statusCode = response.getStatusLine().getStatusCode();
String responseBody = EntityUtils.toString(response.getEntity());
int code = 200;
if (statusCode == code) {
System.out.println("索引删除成功");
} else {
System.out.println("索引删除失败：" + responseBody);
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

59
src/main/java/com/bfd/analyze/test/HttpClientDeleteExample.java
View File

@ -0,0 +1,59 @@
package com.bfd.analyze.test;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import java.io.IOException;
/**
* @author jian.mao
* @date 2023年11月9日
* @description
*/
public class HttpClientDeleteExample {
public static void main(String[] args) {
// 创建 HttpClient 连接池
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
connManager.setMaxTotal(100);
// 设置 Elasticsearch 集群地址
HttpHost[] hosts = {
new HttpHost("172.16.12.55", 9200, "http"),
new HttpHost("172.16.12.56", 9200, "http"),
new HttpHost("172.16.12.57", 9200, "http")
};
// 设置用户凭证
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials("elastic", "bfd123"));
// 创建 HttpClient 实例并配置连接信息
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(connManager)
.setDefaultCredentialsProvider(credentialsProvider)
.build();
// 构造删除文档请求
HttpDelete deleteRequest = new HttpDelete("http://172.16.12.55:9200/analyze_00001/_doc/1");
deleteRequest.setHeader("Content-Type", "application/json");
try {
// 执行删除文档请求
httpClient.execute(deleteRequest);
// 关闭 HttpClient 连接
httpClient.close();
System.out.println("删除文档成功");
} catch (IOException e) {
System.out.println("删除文档出错: " + e.getMessage());
}
}
}

81
src/main/java/com/bfd/analyze/test/IndexDataWithHttpClient.java
View File

@ -0,0 +1,81 @@
package com.bfd.analyze.test;
import java.util.HashMap;
import java.util.Map;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
/**
* @author jian.mao
* @date 2023年11月3日
* @description
*/
public class IndexDataWithHttpClient {
public static void main(String[] args) {
// Elasticsearch索引名称
String indexName = "analyze_00001";
// 创建凭据提供者
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "bfd123"));
// 创建一个 HttpClient，设置凭据提供者
CloseableHttpClient httpClient = HttpClients.custom()
.setDefaultCredentialsProvider(credentialsProvider)
.build();
try {
// 创建一个 HTTP POST 请求用于写入数据到索引
// Elasticsearch主机、索引名称和文档ID
HttpPost httpPost = new HttpPost("http://172.16.12.55:9200/" + indexName + "/_doc/1");
// 设置请求体，包含要写入的文档数据
Map<String, Object> document = new HashMap<String, Object>(16);
document.put("created", "2023-11-02");
document.put("lastEdit", "2023-11-02");
document.put("appId", 1);
document.put("transferId", 2);
document.put("appName", "Youtube采集");
document.put("appCode", "ABC1233");
document.put("appDescribe", "Youtube采集");
document.put("module", "1");
document.put("result", "test");
document.put("nextAppId", new JSONArray());
document.put("startTag", true);
document.put("scenesId", 1);
document.put("businessKey", "B321321y");
StringEntity entity = new StringEntity(JSONObject.toJSONString(document), ContentType.APPLICATION_JSON);
httpPost.setEntity(entity);
// 发送请求并获取响应
HttpResponse response = httpClient.execute( httpPost);
// 处理响应
int statusCode = response.getStatusLine().getStatusCode();
String responseBody = EntityUtils.toString(response.getEntity());
int code = 201;
if (statusCode == code) {
System.out.println("文档写入成功");
} else {
System.out.println("文档写入失败：" + responseBody);
}
httpClient.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}

177
src/main/java/com/bfd/analyze/utils/DateUtil.java
View File

@ -0,0 +1,177 @@
package com.bfd.analyze.utils;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import lombok.extern.slf4j.Slf4j;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
/**
* 日期工具类
*
* @author jian.mao
* @date 2022年11月15日
* @description
*/
@Slf4j
public class DateUtil {
/**
* @return
*/
public static String getTimeStrForNow() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH");
return sdf.format(new Date());
}
public static String getTimeStrForDay(long time) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
return sdf.format(new Date(time * 1000));
}
public static String getTimeStrForDay() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
return sdf.format(new Date());
}
public static String getDateTime() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String time = sdf.format(new Date());
return time;
}
public static String getDateTime(Long timestap) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String time = sdf.format(new Date(timestap));
return time;
}
public static String getDate(Long timestap) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
String time = sdf.format(new Date(timestap));
return time;
}
public static String getDateTimeForMonth() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM");
String time = sdf.format(new Date());
return time;
}
/**
* 休眠
*
* @param millis 毫秒
*/
public static void sleep(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/**
* 1. @Description:时间戳转时间
* 2. @Author: ying.zhao
* 3. @Date: 2023/3/28
*/
public static String timestampToDate(String time) {
int thirteen = 13;
int ten = 10;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// if (time.length() == thirteen) {
if (time.length() > ten) {
return sdf.format(new Date(Long.parseLong(time)));
} else {
return sdf.format(new Date(Integer.parseInt(time) * 1000L));
}
}
public static String parseCreated(String jsonTime){
String formattedDateTime = getDateTime();
try {
// 使用fastjson解析JSON数据
JSONObject jsonObject = JSON.parseObject(jsonTime);
// 获取日期和时间的值
JSONObject dateObject = jsonObject.getJSONObject("date");
int day = dateObject.getIntValue("day");
int month = dateObject.getIntValue("month");
int year = dateObject.getIntValue("year");
JSONObject timeObject = jsonObject.getJSONObject("time");
int hour = timeObject.getIntValue("hour");
int minute = timeObject.getIntValue("minute");
int second = timeObject.getIntValue("second");
// 创建LocalDateTime对象
LocalDateTime dateTime = LocalDateTime.of(year, month, day, hour, minute, second);
// 定义日期时间格式化器
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
// 格式化日期时间
formattedDateTime = dateTime.format(formatter);
} catch (Exception e) {
log.info("日期转换失败：{}",e);
}
return formattedDateTime;
}
/**
* 字符串转换日期
* @param format
* @param date
* @return
*/
public static Date strToDate(String format,String date){
SimpleDateFormat sdf = new SimpleDateFormat(format);
if (date == null || date.equals("")){
return new Date();
}else{
Date ru = null;
try {
ru = sdf.parse(date);
} catch (ParseException e) {
e.printStackTrace();
}
return ru;
}
}
/**
* 日期格式话
* @param format 日期格式
* @param dater 要转换的日期,默认当前时间
* @return
*/
public static String FormatDate(String format,Date date){
String fromatDate = null;
SimpleDateFormat sdf = new SimpleDateFormat(format);
if (date == null){
fromatDate = sdf.format(new Date());
}else{
fromatDate = sdf.format(date);
}
return fromatDate;
}
public static void main(String[] args) {
String time = timestampToDate("955814400000");
System.out.println(time);
}
}

912
src/main/java/com/bfd/analyze/utils/DownLoadUtil.java
View File

@ -0,0 +1,912 @@
package com.bfd.analyze.utils;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.bfd.analyze.entity.Constants;
/**
* @author jian.mao
* @date 2023年12月7日
* @description
*/
public class DownLoadUtil {
private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36";
private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class);
/** 代理服务器(产品官网 www.16yun.cn) **/
final static String PROXYHOST = "u270.40.tp.16yun.cn";
final static Integer PROXYPORT = 6448;
/** 代理验证信息 **/
final static String PROXYUSER = "16HFBVJC";
final static String PROXYPASS = "897944";
private static PoolingHttpClientConnectionManager cm = null;
private static HttpRequestRetryHandler httpRequestRetryHandler = null;
private static HttpHost proxy = null;
private static CredentialsProvider credsProvider = null;
private static RequestConfig reqConfig = null;
static {
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory
.getSocketFactory();
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory
.getSocketFactory();
Registry registry = RegistryBuilder.create().register("http", plainsf)
.register("https", sslsf).build();
cm = new PoolingHttpClientConnectionManager(registry);
cm.setMaxTotal(20);
cm.setDefaultMaxPerRoute(5);
proxy = new HttpHost(PROXYHOST, PROXYPORT, "https");
credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(PROXYUSER, PROXYPASS));
reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000)
.setConnectTimeout(5000).setSocketTimeout(5000)
.setExpectContinueEnabled(false)
.setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build();
}
/**
* 模拟客户端get请求
*
* @param url
* 模拟请求得url
* @param headers
* 头部信息，没有可以不传
* @return
*/
@SafeVarargs
public static String proxyDoGet(String url, Map<String, Object>... headers) {
// 设置超时时间
int timeout = 30;
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(timeout * 1000)
.setConnectionRequestTimeout(timeout * 1000)
.setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(timeout * 1000)
.setTcpNoDelay(true).build();
AuthCache authCache = new BasicAuthCache();
authCache.put(proxy, new BasicScheme());
HttpClientContext localContext = HttpClientContext.create();
localContext.setAuthCache(authCache);
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
CloseableHttpClient httpClient = httpBuilder
.setDefaultSocketConfig(socketConfig)
.setDefaultRequestConfig(config)
.setDefaultCredentialsProvider(credsProvider).build();
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(reqConfig);
if (headers != null && headers.length > 0) {
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpGet.setHeader(key, tempHeaders.get(key).toString());
}
} else {
httpGet.setHeader("Accept",
"application/json, text/javascript, */*; q=0.01");
httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
}
CloseableHttpResponse response = null;
String html = "";
int notFundCode = 404;
int successCode = 200;
try {
response = httpClient.execute(httpGet, localContext);
// 从响应模型中获取响应实体
HttpEntity responseEntity = response.getEntity();
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
if (statusLine.getStatusCode() == successCode) {
if (responseEntity != null) {
html = EntityUtils.toString(responseEntity, "utf-8");
System.out.println("响应内容长度为:"
+ responseEntity.getContentLength());
// 下载结果为空不正常
if (html.equals(Constants.EMPTY)) {
html = "Download failed error is:reslut is null";
}
}
} else if (statusLine.getStatusCode() == notFundCode) {
html = "<h2>页面404，正常结束请求即可</h2>";
} else {
throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:reslut is null";
}finally{
try {
response.close();
httpClient.close();
} catch (Exception e) {
e.printStackTrace();
}
}
return html;
}
public static String httpsslProxyGet(String url, Map<String, Object>... headers) throws Exception {
//采用绕过验证的方式处理https请求
SSLContext sslcontext = createIgnoreVerifySSL();
// 设置协议http和https对应的处理socket链接工厂的对象
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslcontext))
.build();
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
connManager.setMaxTotal(50);
connManager.setDefaultMaxPerRoute(10);
HttpClients.custom().setConnectionManager(connManager);
// 设置超时时间
int timeout = 30;
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(timeout * 1000)
.setConnectionRequestTimeout(timeout * 1000)
.setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(timeout * 1000)
.setTcpNoDelay(true).build();
AuthCache authCache = new BasicAuthCache();
authCache.put(proxy, new BasicScheme());
HttpClientContext localContext = HttpClientContext.create();
localContext.setAuthCache(authCache);
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
CloseableHttpClient httpClient = httpBuilder
.setConnectionManager(connManager)
.setDefaultSocketConfig(socketConfig)
.setDefaultRequestConfig(config)
.setDefaultCredentialsProvider(credsProvider).build();
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(reqConfig);
if (headers != null && headers.length > 0) {
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpGet.setHeader(key, tempHeaders.get(key).toString());
}
} else {
httpGet.setHeader("Accept",
"application/json, text/javascript, */*; q=0.01");
httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
}
CloseableHttpResponse response = null;
String html = "";
int notFundCode = 404;
int successCode = 200;
try {
response = httpClient.execute(httpGet, localContext);
// 从响应模型中获取响应实体
HttpEntity responseEntity = response.getEntity();
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
if (statusLine.getStatusCode() == successCode) {
if (responseEntity != null) {
html = EntityUtils.toString(responseEntity, "utf-8");
System.out.println("响应内容长度为:"
+ responseEntity.getContentLength());
// 下载结果为空不正常
if (html.equals(Constants.EMPTY)) {
html = "Download failed error is:reslut is null";
}
}
} else if (statusLine.getStatusCode() == notFundCode) {
html = "<h2>页面404，正常结束请求即可</h2>";
} else {
throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:reslut is null";
}finally{
try {
response.close();
httpClient.close();
} catch (Exception e) {
e.printStackTrace();
}
}
return html;
}
/**
* json参数方式POST提交
* @param url
* @param params
* @return
*/
public static String doPost(String url, String params){
String strResult = "";
//设置超时时间
int timeout =60;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(timeout * 1000)
.setTcpNoDelay(true).build();
// AuthCache authCache = new BasicAuthCache();
// authCache.put(proxy, new BasicScheme());
// HttpClientContext localContext = HttpClientContext.create();
// localContext.setAuthCache(authCache);
// 1. 获取默认的client实例
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
httpBuilder.setUserAgent(ua);
HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
// HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
// .setDefaultCredentialsProvider(credsProvider).build();
// 2. 创建httppost实例
HttpPost httpPost = new HttpPost(url);
// httpPost.setConfig(reqConfig);
httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
HttpResponse resp = null;
try {
httpPost.setEntity(new StringEntity(params,"utf-8"));
resp = client.execute(httpPost);
// resp = client.execute(httpPost,localContext);
StatusLine statusLine = resp.getStatusLine();
System.out.println("响应状态为:" + resp.getStatusLine());
int successCode = 200;
int errorCode = 404;
if(statusLine.getStatusCode() == successCode){
// 7. 获取响应entity
HttpEntity respEntity = resp.getEntity();
strResult = EntityUtils.toString(respEntity, "UTF-8");
if(strResult.equals("")){
strResult = "Download failed error is:reslut is null";
}
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
return strResult;
}
public static String httpPost(String url,String params) {
String html="";
html = doPost(url,params);
int i = 1;
while(true){
if(html.contains("Download failed error is:")){
log.error("DownLoadUtil------------->download is failure,url is:"+url);
DateUtil.sleep(5000);
i++;
}else{
break;
}
if(i > 5){
break;
}
html = doPost(url,params);
}
return html;
}
/**
* 绕过验证
*
* @return
* @throws NoSuchAlgorithmException
* @throws KeyManagementException
*/
public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
SSLContext sc = SSLContext.getInstance("SSLv3");
// 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
X509TrustManager trustManager = new X509TrustManager() {
@Override
public void checkClientTrusted(
java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
String paramString) throws CertificateException {
}
@Override
public void checkServerTrusted(
java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
String paramString) throws CertificateException {
}
@Override
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
return null;
}
};
sc.init(null, new TrustManager[] { trustManager }, null);
return sc;
}
/**
* 模拟请求
*
* @param url 资源地址
* @param map 参数列表
* @param encoding 编码
* @return
* @throws NoSuchAlgorithmException
* @throws KeyManagementException
* @throws IOException
* @throws ClientProtocolException
*/
public static String httpsslGet(String url,Map<String, Object> ... headers) {
String html="";
CloseableHttpClient client = null;
HttpEntity responseEntity = null;
CloseableHttpResponse response = null;
try {
log.debug("DownLoadUtil------------->设置下载相关信息， start....");
//采用绕过验证的方式处理https请求
SSLContext sslcontext = createIgnoreVerifySSL();
// 设置协议http和https对应的处理socket链接工厂的对象
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslcontext))
.build();
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
connManager.setMaxTotal(50);
connManager.setDefaultMaxPerRoute(10);
HttpClients.custom().setConnectionManager(connManager);
//设置超时时间
int timeout = 30;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(10000)
.setTcpNoDelay(true).build();
// 设置重定向策略
LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy();
//创建自定义的httpclient对象
client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build();
// CloseableHttpClient client = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(url);
if(headers != null && headers.length > 0){
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpGet.setHeader(key,tempHeaders.get(key).toString());
}
}else{
httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
}
log.debug("DownLoadUtil------------->设置下载相关信息， end....");
try {
log.debug("DownLoadUtil------------->下载执行，start....");
httpGet.setConfig(config);
response = client.execute(httpGet);
log.debug("DownLoadUtil------------->下载执行，end....");
// 从响应模型中获取响应实体
StatusLine statusLine = response.getStatusLine();
log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is start ....");
System.out.println("响应状态为:" + response.getStatusLine());
responseEntity = response.getEntity();
log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is end ....");
int successCode = 200;
int errorCode = 404;
if(statusLine.getStatusCode() == successCode){
if (responseEntity != null) {
html=EntityUtils.toString(responseEntity,"utf-8");
System.out.println("响应内容长度为:" + responseEntity.getContentLength());
}
}else if(statusLine.getStatusCode() == errorCode){
html = "<h2>页面404，正常结束请求即可</h2>";
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}finally{
try {
responseEntity.getContent().close();
response.close();
client.close();
} catch (Exception e) {
e.printStackTrace();
}
}
return html;
}
public static String httpSSLGet(String url,Map<String, Object> ... headers) {
String html="";
html = httpsslGet(url,headers);
int i = 1;
while(true){
if(html.contains("Download failed error is:")){
log.error("DownLoadUtil------------->download is failure,url is:"+url);
DateUtil.sleep(30000);
i++;
}else{
break;
}
if(i > 5){
break;
}
html = httpsslGet(url,headers);
}
return html;
}
public static String doPostFrom(String url,Map<String, Object> param,Map<String, Object> ... headers){
//设置超时时间
int timeout = 15;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(10000)
.setTcpNoDelay(true).build();
// AuthCache authCache = new BasicAuthCache();
// authCache.put(proxy, new BasicScheme());
// HttpClientContext localContext = HttpClientContext.create();
// localContext.setAuthCache(authCache);
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
httpBuilder.setUserAgent(ua);
// HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
// .setDefaultCredentialsProvider(credsProvider).build();
HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
HttpPost httpPost = new HttpPost(url);
// httpPost.setConfig(reqConfig);
if(headers != null && headers.length > 0){
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpPost.setHeader(key,tempHeaders.get(key).toString());
}
}else{
httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36");
// httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html");
}
// 创建请求参数
List<NameValuePair> list = new LinkedList<>();
for (String key : param.keySet()) {
BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
list.add(param1);
}
// 使用URL实体转换工具
String html="";
try {
UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
httpPost.setEntity(entityParam);
HttpResponse response = httpClient.execute(httpPost);
// HttpResponse response = httpClient.execute(httpPost,localContext);
// 从响应模型中获取响应实体
HttpEntity responseEntity = response.getEntity();
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
int successCode = 200;
int errorCode = 404;
if(statusLine.getStatusCode() == successCode){
if (responseEntity != null) {
html=EntityUtils.toString(responseEntity,"utf-8");
}
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
return html;
}
public static String httpPostForm(String url,Map<String,Object> params,Map<String, Object> ... headers) {
String html="";
html = doPostFrom(url,params);
int i = 1;
while(true){
if(html.contains("Download failed error is:")){
log.error("DownLoadUtil------------->download is failure,url is:"+url);
DateUtil.sleep(5000);
i++;
}else{
break;
}
if(i > 5){
break;
}
html = doPostFrom(url,params,headers);
}
return html;
}
public static String dosslPost(String url,String params,Map<String, Object> ... headers) {
String html="";
CloseableHttpClient client = null;
HttpEntity responseEntity = null;
CloseableHttpResponse response = null;
try {
//采用绕过验证的方式处理https请求
SSLContext sslcontext = createIgnoreVerifySSL();
// 设置协议http和https对应的处理socket链接工厂的对象
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslcontext))
.build();
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
HttpClients.custom().setConnectionManager(connManager);
//设置超时时间
int timeout = 5;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(10000)
.setTcpNoDelay(true).build();
//创建自定义的httpclient对象
client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
// CloseableHttpClient client = HttpClients.createDefault();
// 2. 创建httppost实例
HttpPost httpPost = new HttpPost(url);
// httpPost.setConfig(reqConfig);
httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
if(headers != null && headers.length > 0){
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpPost.setHeader(key,tempHeaders.get(key).toString());
}
}else{
httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
}
try {
httpPost.setEntity(new StringEntity(params,"utf-8"));
response = client.execute(httpPost);
// 从响应模型中获取响应实体
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
responseEntity = response.getEntity();
int successCode = 200;
int errorCode = 404;
if(statusLine.getStatusCode() == successCode){
if (responseEntity != null) {
html=EntityUtils.toString(responseEntity,"utf-8");
System.out.println("响应内容长度为:" + responseEntity.getContentLength());
}
}else if(statusLine.getStatusCode() == errorCode){
html = "<h2>页面404，正常结束请求即可</h2>";
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}finally{
try {
responseEntity.getContent().close();
response.close();
client.close();
} catch (UnsupportedOperationException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
return html;
}
public static String dosslPostForm(String url,Map<String, Object> param,Map<String, Object> ... headers) {
String html="";
try {
//采用绕过验证的方式处理https请求
SSLContext sslcontext = createIgnoreVerifySSL();
// 设置协议http和https对应的处理socket链接工厂的对象
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslcontext))
.build();
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
HttpClients.custom().setConnectionManager(connManager);
//设置超时时间
int timeout = 5;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(10000)
.setTcpNoDelay(true).build();
//创建自定义的httpclient对象
CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
// CloseableHttpClient client = HttpClients.createDefault();
// 2. 创建httppost实例
HttpPost httpPost = new HttpPost(url);
// httpPost.setConfig(reqConfig);
if(headers != null && headers.length > 0){
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpPost.setHeader(key,tempHeaders.get(key).toString());
}
}else{
httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
}
// 创建请求参数
List<NameValuePair> list = new LinkedList<>();
for (String key : param.keySet()) {
BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
list.add(param1);
}
// 使用URL实体转换工具
try {
UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
httpPost.setEntity(entityParam);
HttpResponse response = client.execute(httpPost);
// HttpResponse response = httpClient.execute(httpPost,localContext);
// 从响应模型中获取响应实体
HttpEntity responseEntity = response.getEntity();
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
int successCode = 200;
int errorCode = 404;
if(statusLine.getStatusCode() == successCode){
if (responseEntity != null) {
html=EntityUtils.toString(responseEntity,"utf-8");
}
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
return html;
}
public static String httpSSLPostForm(String url,Map<String, Object> params,Map<String, Object> ...headers) {
String html="";
try {
html = dosslPostForm(url,params,headers);
} catch (Exception e) {
e.printStackTrace();
// TODO: handle exception
html = "Download failed error is:Exception!";
}
int i = 1;
while(true){
if(html.contains("Download failed error is:")){
log.error("DownLoadUtil------------->download is failure,url is:"+url);
DateUtil.sleep(30000);
i++;
}else{
break;
}
if(i > 5){
break;
}
try {
html = dosslPostForm(url,params,headers);
} catch (Exception e) {
e.printStackTrace();
// TODO: handle exception
html = "Download failed error is:Exception!";
}
}
return html;
}
public static String httpSSLPost(String url,String params,Map<String, Object> ...headers) {
String html="";
try {
html = dosslPost(url,params,headers);
} catch (Throwable e) {
e.printStackTrace();
// TODO: handle exception
html = "Download failed error is:Exception!";
}
int i = 1;
while(true){
if(html.contains("Download failed error is:")){
log.error("DownLoadUtil------------->download is failure,url is:"+url);
DateUtil.sleep(30000);
i++;
}else{
break;
}
if(i > 5){
break;
}
try {
html = dosslPost(url,params,headers);
} catch (Throwable e) {
e.printStackTrace();
// TODO: handle exception
html = "Download failed error is:Exception!";
}
}
return html;
}
/**
* 模拟客户端get请求
* @param url 模拟请求得url
* @param headers 头部信息，没有可以不传
* @return
*/
public static String doGet(String url,Map<String, Object> ... headers){
//设置超时时间
int timeout = 15;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(10000)
.setTcpNoDelay(true).build();
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
httpBuilder.setUserAgent(ua);
HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
HttpGet httpGet = new HttpGet(url);
if(headers != null && headers.length > 0){
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpGet.setHeader(key,tempHeaders.get(key).toString());
}
}else{
httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
}
String html="";
try {
HttpResponse response = httpClient.execute(httpGet);
// 从响应模型中获取响应实体
HttpEntity responseEntity = response.getEntity();
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
int successCode = 200;
int errorCode = 404;
if(statusLine.getStatusCode() == successCode){
if (responseEntity != null) {
html=EntityUtils.toString(responseEntity,"utf-8");
if(html.equals("")){
html = "Download failed error is:reslut is null";
}
}
}else if(statusLine.getStatusCode() == errorCode){
html = "<h2>页面404，正常结束请求即可</h2>";
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
return html;
}
// public static void main(String[] args) throws Exception {
// Map<String, Object> headers = new HashMap<String, Object>();
// headers.put("accept", "application/json, text/plain, */*");
// headers.put("authority", "mp.weixin.qq.com");
// headers.put("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
// headers.put("accept-language", "zh-CN,zh;q=0.9,en;q=0.8");
// headers.put("user-agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36");
// String url = "https://mp.weixin.qq.com/s?__biz=MzU5NDgzNTg2OQ==&mid=2247528413&idx=2&sn=cc79680d6f6cc160e7aa3f5d2101f2ae";
// String html = httpsslProxyGet(url, headers);
// System.out.println(html);
// }
}

408
src/main/java/com/bfd/analyze/utils/ElasticsearchUtil.java
View File

@ -0,0 +1,408 @@
package com.bfd.analyze.utils;
import com.alibaba.fastjson.JSONObject;
import com.bfd.analyze.entity.Constants;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
/**
* es操作工具类
*
* @author jian.mao
* @date 2024年6月6日
* @description
*/
@Slf4j
public class ElasticsearchUtil {
/**
* es根据条件更新
*
* @param index
* 索引
* @param username
* 用户名
* @param password
* 密码
* @param field
* 更新字段名称
* @param fieldValue
* 更新字段值
* @param conditionField
* 条件字段名称
* @param conditionValue
* 条件值
* @return
* @throws IOException
*/
public static String updateFieldByCondition(String index, String username, String password, String field,
Object fieldValue, String conditionField, Object conditionValue,String nodes)
throws IOException {
//只写入一个节点
String node = nodes.split(",")[0];
// 构造请求 URL
StringBuffer url = new StringBuffer("http://");
url.append(node);
url.append("/");
url.append(index);
url.append("/_update_by_query?refresh=true");
// Script part
Map<String, Object> params = new HashMap<>(16);
params.put("value", fieldValue);
Map<String, Object> script = new HashMap<>(16);
script.put("source",
String.format("ctx._source.%s = params.value", field));
script.put("lang", "painless");
script.put("params", params);
// Query part
Map<String, Object> match = new HashMap<>(16);
match.put(conditionField, conditionValue);
Map<String, Object> query = new HashMap<>(16);
query.put("match", match);
// Combine script and query
Map<String, Object> requestBody = new HashMap<>(16);
requestBody.put("script", script);
requestBody.put("query", query);
// Convert Map to JSON string
String jsonBody = JSONObject.toJSONString(requestBody);
return sendPostRequest(url.toString(), jsonBody, username, password);
}
/**
* @param index 索引
* @param id 主键
* @param username 用户名
* @param password 密码
* @param field 更新字段
* @param fieldValue 字段值
* @param nodes 节点信息
* @return
* @throws IOException
*/
public static String updateFieldById(String index, String id, String username, String password, String field,
Object fieldValue, String nodes) throws IOException {
// 只写入一个节点
String node = nodes.split(",")[0];
// 构造请求 URL
String url = String.format("http://%s/%s/_update/%s", node, index, id);
// 构造请求脚本
Map<String, Object> params = new HashMap<String, Object>(16);
params.put("value", fieldValue);
Map<String, Object> scriptContent = new HashMap<String, Object>(16);
scriptContent.put("source", String.format("ctx._source.%s = params.value", field));
scriptContent.put("lang", "painless");
scriptContent.put("params", params);
// 将 Script 内容嵌套在脚本对象中
Map<String, Object> script = new HashMap<String, Object>(16);
script.put("script", scriptContent);
// 将 Map 转换为 JSON 字符串
String jsonBody = JSONObject.toJSONString(script);
// 发送 POST 请求
return sendPostRequest(url, jsonBody, username, password);
}
/**
* rest接口请求es
*
* @param url
* @param jsonBody
* @param username
* @param password
* @return
* @throws IOException
*/
private static String sendPostRequest(String url, String jsonBody,
String username, String password) throws IOException {
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
HttpPost httpPost = new HttpPost(url);
// Set the content type to JSON
httpPost.setHeader("Content-Type", "application/json");
// Set the basic authentication header
String auth = username + ":" + password;
byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes());
String authHeader = "Basic " + new String(encodedAuth);
httpPost.setHeader("Authorization", authHeader);
// Set the JSON body
StringEntity entity = new StringEntity(jsonBody);
httpPost.setEntity(entity);
// Execute the request
return EntityUtils.toString(httpClient.execute(httpPost)
.getEntity());
}
}
/**
* 单条件查询
* @param conditionField
* @param conditionValue
* @param nodes
* @param username
* @param password
* @param indexName
* @return
* @throws IOException
*/
public static String getDocumentBySingleCondition(String conditionField, Object conditionValue,String nodes,String username,String password,String indexName) throws IOException {
//只写入一个节点
String node = nodes.split(",")[0];
// 创建凭据提供者
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
// 创建一个 HttpClient，设置凭据提供者
try (CloseableHttpClient httpClient = HttpClients.custom()
.setDefaultCredentialsProvider(credentialsProvider)
.build()) {
// 构造请求 URL
StringBuffer host = new StringBuffer("http://");
host.append(node);
host.append("/");
host.append(indexName);
host.append("/_search");
// 构造请求体
String requestBody = String.format("{ \"query\": { \"match\": { \"%s\": \"%s\" }}}",conditionField, conditionValue);
// 创建 HTTP POST 请求
HttpPost httpPost = new HttpPost(host.toString());
httpPost.setHeader("Content-Type", "application/json");
httpPost.setEntity(new StringEntity(requestBody));
// 发送请求并获取响应
HttpResponse response = httpClient.execute(httpPost);
// 解析响应
HttpEntity entity = response.getEntity();
if (entity != null) {
String responseString = EntityUtils.toString(entity);
log.info("条件：{}，条件查询响应体：{}" ,requestBody,responseString);
return responseString;
} else {
log.warn("条件查询异常");
return Constants.EMPTY;
}
}
}
/**
* 查询数据，自定义请求体方法
* @param nodes
* @param username
* @param password
* @param indexName
* @param requestBody
* @return
* @throws IOException
*/
public static String getDocument(String nodes,String username,String password,String indexName,String requestBody) throws IOException {
//只写入一个节点
String node = nodes.split(",")[0];
// 创建凭据提供者
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
// 创建一个 HttpClient，设置凭据提供者
try (CloseableHttpClient httpClient = HttpClients.custom()
.setDefaultCredentialsProvider(credentialsProvider)
.build()) {
// 构造请求 URL
StringBuffer host = new StringBuffer("http://");
host.append(node);
host.append("/");
host.append(indexName);
host.append("/_search");
// 构造请求体
// 创建 HTTP POST 请求
HttpPost httpPost = new HttpPost(host.toString());
httpPost.setHeader("Content-Type", "application/json");
httpPost.setEntity(new StringEntity(requestBody));
// 发送请求并获取响应
HttpResponse response = httpClient.execute(httpPost);
// 解析响应
HttpEntity entity = response.getEntity();
if (entity != null) {
String responseString = EntityUtils.toString(entity);
log.info("条件查询响应体：{}" ,responseString);
return responseString;
} else {
log.warn("条件查询异常");
return Constants.EMPTY;
}
}
}
/**
* 条件删除，含version
* @param conditionField
* @param conditionValue
* @param nodes
* @param username
* @param password
* @param indexName
* @param version
* @return
*/
public static String deleteDataBycondition(String conditionField, Object conditionValue,String nodes,String username,String password,String indexName,String version){
//只写入一个节点
String node = nodes.split(",")[0];
// 创建凭据提供者
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
// 创建一个 HttpClient，设置凭据提供者
try (CloseableHttpClient httpClient = HttpClients.custom()
.setDefaultCredentialsProvider(credentialsProvider)
.build()) {
// 构造请求 URL
StringBuffer host = new StringBuffer("http://");
host.append(node);
host.append("/");
host.append(indexName);
host.append("/_delete_by_query?refresh=true");
// 构造请求体
// String requestBody = String.format("{ \"query\": { \"match\": { \"%s\": \"%s\" }}}",conditionField, conditionValue);
//删除条件 添加version
String requestBody = String.format("{\"query\":{\"bool\":{\"must\":[{\"match\":{\"%s\":\"%s\"}}],\"must_not\":[{\"match\":{\"version\":\"%s\"}}]}}}",conditionField, conditionValue,version);
// 创建 HTTP POST 请求
HttpPost httpPost = new HttpPost(host.toString());
httpPost.setHeader("Content-Type", "application/json");
httpPost.setEntity(new StringEntity(requestBody));
// 发送请求并获取响应
HttpResponse response = httpClient.execute(httpPost);
// 解析响应
HttpEntity entity = response.getEntity();
if (entity != null) {
String responseString = EntityUtils.toString(entity);
log.info("条件：{}，有版本号删除响应体：{}" ,requestBody, responseString);
return responseString;
} else {
log.warn("有版本号删除异常");
return Constants.EMPTY;
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* 条件删除，不含version
* @param conditionField
* @param conditionValue
* @param nodes
* @param username
* @param password
* @param indexName
* @return
*/
public static String deleteDataByconditionNoVersion(String conditionField, Object conditionValue,String nodes,String username,String password,String indexName,String searchType){
//只写入一个节点
String node = nodes.split(",")[0];
// 创建凭据提供者
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
// 创建一个 HttpClient，设置凭据提供者
try (CloseableHttpClient httpClient = HttpClients.custom()
.setDefaultCredentialsProvider(credentialsProvider)
.build()) {
// 构造请求 URL
StringBuffer host = new StringBuffer("http://");
host.append(node);
host.append("/");
host.append(indexName);
host.append("/_delete_by_query?refresh=true");
// 构造请求体
String requestBody = String.format("{ \"query\": { \"%s\": { \"%s\": \"%s\" }}}",searchType,conditionField, conditionValue);
// 创建 HTTP POST 请求
HttpPost httpPost = new HttpPost(host.toString());
httpPost.setHeader("Content-Type", "application/json");
httpPost.setEntity(new StringEntity(requestBody));
// 发送请求并获取响应
HttpResponse response = httpClient.execute(httpPost);
// 解析响应
HttpEntity entity = response.getEntity();
if (entity != null) {
String responseString = EntityUtils.toString(entity);
log.info("条件：{}，无版本号删除响应体：{}",requestBody,responseString);
return responseString;
} else {
log.warn("无版本号删除异常");
return Constants.EMPTY;
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* 根据主键删除
* @param id
* @param nodes
* @param username
* @param password
* @param indexName
* @return
*/
public static String deleteDataById(String id, String nodes, String username, String password, String indexName) {
// 只写入一个节点
String node = nodes.split(",")[0];
// 创建凭据提供者
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
// 创建一个 HttpClient，设置凭据提供者
try (CloseableHttpClient httpClient = HttpClients.custom()
.setDefaultCredentialsProvider(credentialsProvider)
.build()) {
// 构造请求 URL
StringBuilder host = new StringBuilder("http://");
host.append(node);
host.append("/");
host.append(indexName);
host.append("/_doc/");
host.append(id);
// 创建 HTTP DELETE 请求
HttpDelete httpDelete = new HttpDelete(host.toString());
// 发送请求并获取响应
HttpResponse response = httpClient.execute(httpDelete);
// 解析响应
HttpEntity entity = response.getEntity();
if (entity != null) {
String responseString = EntityUtils.toString(entity);
log.info("主键删除响应体：{}",responseString);
return responseString;
} else {
log.warn("主键删除异常");
return Constants.EMPTY;
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static void main(String[] args) {
String id = "fb2163f8-b1ee-4f96-8919-12b45d4a0df1";
String nodes = "172.18.1.147:9200";
String username = "elastic";
String password = "baifendian";
String indexName = "dispatch_analyze_00001";
deleteDataById(id, nodes, username, password, indexName);
}
}

27
src/main/java/com/bfd/analyze/utils/EncryptionUtil.java
View File

@ -0,0 +1,27 @@
package com.bfd.analyze.utils;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
/**
* @author jian.mao
* @date 2023年3月10日
* @description
*/
public class EncryptionUtil {
public static String md5(String text) {
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(text.getBytes());
byte[] bytes = md.digest();
StringBuilder sb = new StringBuilder();
for (byte b : bytes) {
sb.append(String.format("%02x", b & 0xff));
}
return sb.toString();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
return null;
}
}
}

30
src/main/java/com/bfd/analyze/utils/FileUtil.java
View File

@ -0,0 +1,30 @@
package com.bfd.analyze.utils;
import java.io.FileWriter;
import java.io.IOException;
/**
* 文件工具类
* @author jian.mao
* @date 2023年7月14日
* @description
*/
public class FileUtil {
/**
* 数据写入文件
* @param Path 文件路径
* @param result 数据
* @throws IOException
*/
public static void writeFile(String path,String result){
try {
FileWriter fw = new FileWriter(path,true);
fw.write(result+"\n");
fw.flush();
fw.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}

39
src/main/java/com/bfd/analyze/utils/JsonUtil.java
View File

@ -0,0 +1,39 @@
package com.bfd.analyze.utils;
import java.util.List;
import java.util.Map;
import com.alibaba.fastjson.JSONObject;
import com.bfd.analyze.entity.Constants;
/**
* json工具
* @author jian.mao
* @date 2023年7月10日
* @description
*/
public class JsonUtil {
/**
* 校验字符串是list/map/str
* @param jsonString
* @return
*/
public static String checkJsonType(String jsonString) {
try {
JSONObject.parseObject(jsonString);
return Constants.MAP_TYPE;
} catch (Exception e) {
try {
List list = JSONObject.parseArray(jsonString);
if(list != null && list.size() > 0 ){
return Constants.LIST_TYPE;
}else{
throw new Exception();
}
} catch (Exception ex) {
return Constants.STRING_TYPE;
}
}
}
}

32
src/main/java/com/bfd/analyze/utils/OtherUtils.java
View File

@ -0,0 +1,32 @@
package com.bfd.analyze.utils;
import java.security.MessageDigest;
/**
* @author jian.mao
* @date 2023年10月7日
* @description
*/
public class OtherUtils {
public static String getMd5(String string) {
try {
MessageDigest md5 = MessageDigest.getInstance("MD5");
byte[] bs = md5.digest(string.getBytes("UTF-8"));
StringBuilder sb = new StringBuilder(40);
for (byte x : bs) {
if ((x & 0xff) >> 4 == 0) {
sb.append("0").append(Integer.toHexString(x & 0xff));
} else {
sb.append(Integer.toHexString(x & 0xff));
}
}
return sb.toString();
} catch (Exception e) {
return "nceaform" + System.currentTimeMillis();
}
}
}

69
src/main/java/com/bfd/analyze/utils/PauseTool.java
View File

@ -0,0 +1,69 @@
package com.bfd.analyze.utils;
import com.alibaba.fastjson.JSON;
import com.bfd.analyze.entity.Constants;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.Set;
/**
* @author:jinming
* @className:ZookeeperNodeMonitor
* @version:1.0
* @description: Zookeeper节点监听和Redis初始化工具类
* @Date:2024/7/2 14:20
*/
@Component
@Slf4j
public class PauseTool {
/**
* 本地缓存
*/
public static final HashMap<String, Object> CACHE = new HashMap<>();
@Autowired
private StringRedisTemplate stringRedisTemplate;
/**
* 初始化Redis中的version_*键并加载到本地缓存
*/
public void initializeRedisCache(StringRedisTemplate stringRedisTemplate) {
try {
Set<String> keys = stringRedisTemplate.keys("version_*");
if (keys != null) {
for (String key : keys) {
String value = stringRedisTemplate.opsForValue().get(key);
if (value != null) {
String sincesId = key.split("_")[1];
CACHE.put(sincesId.concat("_").concat(value), value);
}
}
}
log.info("当前缓存version信息:{}", JSON.toJSON(CACHE));
} catch (Exception e) {
log.error("Error initializing Redis cache", e);
}
}
/**
* 校验 任务是否暂停
* @param scenesId
* @param version
* @return
*/
public Boolean check(int scenesId,int version){
String scenesVsersion = stringRedisTemplate.opsForValue().get(Constants.VERSION + Constants.UNDERLINE + scenesId);
int scenes_version = Integer.parseInt(scenesVsersion);
if (scenes_version==version){
return true;
}else {
return false;
}
}
}

107
src/main/java/com/bfd/analyze/utils/QueueUtil.java
View File

@ -0,0 +1,107 @@
package com.bfd.analyze.utils;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
/**
* 队列工具类
* @author jian.mao
* @date 2023年3月22日
* @description
*/
public class QueueUtil {
/****任务****/
public static LinkedBlockingDeque<Map<String, Object>> taskQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****详情任务队列***/
public static LinkedBlockingDeque<Map<String, Object>> taskDetailQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****股权出质****/
public static LinkedBlockingDeque<Map<String, Object>> gqczQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> gqczErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****动产抵押****/
public static LinkedBlockingDeque<Map<String, Object>> dcdyQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> dcdyErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****对外担保****/
public static LinkedBlockingDeque<Map<String, Object>> dydbQueue = new LinkedBlockingDeque<Map<String, Object>>();
/**分支机构**/
public static LinkedBlockingDeque<Map<String, Object>> fzjgQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> fzjgErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/***对外担保***/
public static LinkedBlockingDeque<Map<String, Object>> dwdbQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> dwdbErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****网站备案***/
public static LinkedBlockingDeque<Map<String, Object>> wzbaQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> wzbaErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****经营异常****/
public static LinkedBlockingDeque<Map<String, Object>> jyycQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> jyycErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/***欠税公告***/
public static LinkedBlockingDeque<Map<String, Object>> qsggQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> qsggErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/***工商变更***/
public static LinkedBlockingDeque<Map<String, Object>> gsbgQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> gsbgErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****专利信息****/
public static LinkedBlockingDeque<Map<String, Object>> zlxxQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> zlxxErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****税收违法****/
public static LinkedBlockingDeque<Map<String, Object>> sswfQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> sswfErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****涉诉公告-法律诉讼****/
public static LinkedBlockingDeque<Map<String, Object>> ssggQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> ssggErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****疑似实际控股人****/
public static LinkedBlockingDeque<Map<String, Object>> yssjkgrQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> yssjkgrErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****舆情信息****/
public static LinkedBlockingDeque<Map<String, Object>> yqxxQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> yqxxErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> yqxxEveryDayQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> yqxxEveryDayDetailQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> yqxxEveryDayErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****司法协助****/
public static LinkedBlockingDeque<Map<String, Object>> sfxzQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> sfxzErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****开庭公告****/
public static LinkedBlockingDeque<Map<String, Object>> ktggQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> ktggErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****被执行人****/
public static LinkedBlockingDeque<Map<String, Object>> bzxrQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> bzxrErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****失信被执行人****/
public static LinkedBlockingDeque<Map<String, Object>> sxbzxrQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> sxbzxrErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****限制消费令****/
public static LinkedBlockingDeque<Map<String, Object>> xzxflQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> xzxflErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****终本案件****/
public static LinkedBlockingDeque<Map<String, Object>> zbajQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> zbajErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****终本案件****/
public static LinkedBlockingDeque<Map<String, Object>> laxxQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> laxxErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****主要人员****/
public static LinkedBlockingDeque<Map<String, Object>> zyryQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> zyryErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****股东信息****/
public static LinkedBlockingDeque<Map<String, Object>> gdxxQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> gdxxErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/****工商信息****/
public static LinkedBlockingDeque<Map<String, Object>> gsxxQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<Map<String, Object>> gsxxErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
/***任务错误队列**/
public static LinkedBlockingDeque<Map<String, Object>> taskErrorQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static Map<String, Object> getATask(
LinkedBlockingDeque<Map<String, Object>> taskQueue) {
Map<String, Object> task;
try {
task = taskQueue.take();
return task;
} catch (Exception e) {
return null;
}
}
}

48
src/main/java/com/bfd/analyze/utils/SpringBootKafka.java
View File

@ -0,0 +1,48 @@
package com.bfd.analyze.utils;
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.stereotype.Component;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.ListenableFutureCallback;
/**
* @PROJECT_NAME: companybusinesscrawl
* @DESCRIPTION:SpringBootKafka 工具类
* @AUTHOR: ying.zhao
* @DATE: 2023/4/6 11:09
*/
@Slf4j
@Component
public class SpringBootKafka {
@Autowired
private KafkaTemplate<String, Object> kafkaTemplate;
/**
* 自定义topicKafkaTemplate
*/
/**
* public static final String TOPIC = "companyBussTest";
**/
public void send(String topic, String message) {
String obj2String = JSONObject.toJSONString(message);
log.info("准备发送消息为：{}", obj2String);
//发送消息
ListenableFuture<SendResult<String, Object>> future = kafkaTemplate.send(topic, message);
future.addCallback(new ListenableFutureCallback<SendResult<String, Object>>() {
@Override
public void onFailure(Throwable throwable) {
//发送失败的处理
log.info(topic + " - 生产者 发送消息失败：" + throwable.getMessage());
}
@Override
public void onSuccess(SendResult<String, Object> stringObjectSendResult) {
//成功的处理
log.info(topic + " - 生产者 发送消息成功：" + stringObjectSendResult.toString());
}
});
}
}

226
src/main/java/com/bfd/analyze/utils/SqlStatementInterceptor.java
View File

@ -0,0 +1,226 @@
package com.bfd.analyze.utils;
//package com.bfd.analyze.utils;
//
//import java.sql.Connection;
//import java.text.DateFormat;
//import java.util.Date;
//import java.util.List;
//import java.util.Locale;
//import java.util.Properties;
//import java.util.regex.Matcher;
//
//import com.alibaba.fastjson.JSONObject;
//import lombok.extern.slf4j.Slf4j;
//import org.apache.commons.collections.CollectionUtils;
//import org.apache.ibatis.executor.statement.StatementHandler;
//import org.apache.ibatis.mapping.BoundSql;
//import org.apache.ibatis.mapping.MappedStatement;
//import org.apache.ibatis.mapping.ParameterMapping;
//import org.apache.ibatis.plugin.Interceptor;
//import org.apache.ibatis.plugin.Intercepts;
//import org.apache.ibatis.plugin.Invocation;
//import org.apache.ibatis.plugin.Plugin;
//import org.apache.ibatis.plugin.Signature;
//import org.apache.ibatis.reflection.DefaultReflectorFactory;
//import org.apache.ibatis.reflection.MetaObject;
//import org.apache.ibatis.reflection.SystemMetaObject;
//import org.apache.ibatis.session.Configuration;
//import org.apache.ibatis.type.TypeHandlerRegistry;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.kafka.core.KafkaTemplate;
//import org.springframework.kafka.support.SendResult;
//import org.springframework.stereotype.Component;
//import org.springframework.util.concurrent.ListenableFuture;
//import org.springframework.util.concurrent.ListenableFutureCallback;
//
//import javax.annotation.PostConstruct;
///**
// * @author:zhaoying
// * @className:SqlStatementInterceptor
// * @version:1.0
// * @description:
// * @Date:2023-04-03 14:34:27
// */
///**
// * 拦截StatementHandler类中参数类型为Statement的 prepare 方法
// * 即拦截 Statement prepare(Connection var1, Integer var2) 方法
//
// */
//
//@Component
//@Slf4j
//@Intercepts({@Signature(type = StatementHandler.class, method = "prepare", args = {Connection.class, Integer.class})})
//public class SqlStatementInterceptor implements Interceptor {
// @Autowired
// private KafkaTemplate<String, Object> kafkaTemplate;
// @Override
// public Object intercept(Invocation invocation) throws Throwable {
//
// StatementHandler statementHandler = (StatementHandler) invocation.getTarget();
// //通过MetaObject优雅访问对象的属性，这里是访问statementHandler的属性;：MetaObject是Mybatis提供的一个用于方便、
// //优雅访问对象属性的对象，通过它可以简化代码、不需要try/catch各种reflect异常，同时它支持对JavaBean、Collection、Map三种类型对象的操作。
// MetaObject metaObject = MetaObject
// .forObject(statementHandler, SystemMetaObject.DEFAULT_OBJECT_FACTORY, SystemMetaObject.DEFAULT_OBJECT_WRAPPER_FACTORY,
// new DefaultReflectorFactory());
// //先拦截到RoutingStatementHandler，里面有个StatementHandler类型的delegate变量，其实现类是BaseStatementHandler，然后就到BaseStatementHandler的成员变量mappedStatement
// MappedStatement mappedStatement = (MappedStatement) metaObject.getValue("delegate.mappedStatement");
// //id为执行的mapper方法的全路径名，如com.uv.dao.UserMapper.insertUser
// String id = mappedStatement.getId();
// log.info("id ==> " + id);
// //sql语句类型 select、delete、insert、update
// String sqlCommandType = mappedStatement.getSqlCommandType().toString();
// log.info("类型 ==> " + sqlCommandType);
//
// BoundSql boundSql = statementHandler.getBoundSql();
// log.info("boundSql={}",boundSql);
// // 获取节点的配置
// Configuration configuration = mappedStatement.getConfiguration();
// // 获取到最终的sql语句
// String newsql = getSql(configuration, boundSql, id);
// log.info("拦截的sql ==>: " + newsql);
// String insertSql = "INSERT INTO";
// if(newsql.contains(insertSql)){
// log.info("是insert语句，发送kafka");
// newsql = newsql.substring(newsql.indexOf(":")+1);
// send(newsql);
// }
//
// long start = System.currentTimeMillis();
// Object returnValue = invocation.proceed();
// long end = System.currentTimeMillis();
// long time = (end - start);
// log.info("sql耗时 ==>: " + time);
// return returnValue;
// //return null;
// }
//
// @Override
// public Object plugin(Object target) {
// return Plugin.wrap(target, this);
// }
//
// @Override
// public void setProperties(Properties properties) {
// }
//
//
// /**
// * 封装了一下sql语句，
// * 使得结果返回完整xml路径下的sql语句节点id + sql语句
// *
// * @param configuration
// * @param boundSql
// * @param sqlId
// * @return
// */
// private String getSql(Configuration configuration, BoundSql boundSql, String sqlId) {
// String sql = showSql(configuration, boundSql);
// StringBuilder str = new StringBuilder(100);
// str.append(sqlId);
// str.append(":");
// str.append(sql);
// return str.toString();
// }
//
// /**
// * 如果参数是String，则添加单引号， 如果是日期，则转换为时间格式器并加单引号；
// * 对参数是null和不是null的情况作了处理<br>
// *
// * @param obj
// * @return
// */
// private String getParameterValue(Object obj) {
// String value = null;
// if (obj instanceof String) {
// value = "'" + obj.toString() + "'";
// } else if (obj instanceof Date) {
// DateFormat formatter = DateFormat.getDateTimeInstance(DateFormat.DEFAULT, DateFormat.DEFAULT, Locale.CHINA);
// value = "'" + formatter.format(new Date()) + "'";
// } else {
// if (obj != null) {
// value = obj.toString();
// } else {
// value = "";
// }
//
// }
// return value;
// }
//
// /**
// * 进行？的替换
// * @param configuration
// * @param boundSql
// * @return
// */
// public String showSql(Configuration configuration, BoundSql boundSql) {
// // 获取参数
// Object parameterObject = boundSql.getParameterObject();
// List<ParameterMapping> parameterMappings = boundSql
// .getParameterMappings();
// // sql语句中多个空格都用一个空格代替
// String sql = boundSql.getSql().replaceAll("[\\s]+", " ");
// if (CollectionUtils.isNotEmpty(parameterMappings) && parameterObject != null) {
// // 获取类型处理器注册器，类型处理器的功能是进行java类型和数据库类型的转换　　　　　　　
// // 如果根据parameterObject.getClass(）可以找到对应的类型，则替换
// TypeHandlerRegistry typeHandlerRegistry = configuration.getTypeHandlerRegistry();
// if (typeHandlerRegistry.hasTypeHandler(parameterObject.getClass())) {
// sql = sql.replaceFirst("\\?", Matcher.quoteReplacement(getParameterValue(parameterObject)));
//
// } else {
// //MetaObject主要是封装了originalObject对象，
// // 提供了get和set的方法用于获取和设置originalObject的属性值,
// // 主要支持对JavaBean、Collection、Map三种类型对象的操作
// MetaObject metaObject = configuration.newMetaObject(parameterObject);
// for (ParameterMapping parameterMapping : parameterMappings) {
// String propertyName = parameterMapping.getProperty();
// if (metaObject.hasGetter(propertyName)) {
// Object obj = metaObject.getValue(propertyName);
// sql = sql.replaceFirst("\\?", Matcher.quoteReplacement(getParameterValue(obj)));
// } else if (boundSql.hasAdditionalParameter(propertyName)) {
// // 该分支是动态sql
// Object obj = boundSql.getAdditionalParameter(propertyName);
// sql = sql.replaceFirst("\\?", Matcher.quoteReplacement(getParameterValue(obj)));
//
// } else {
// //打印出缺失，提醒该参数缺失并防止错位
// sql = sql.replaceFirst("\\?", "缺失");
// }
// }
// }
// }
// return sql;
// }
//// @Autowired
//// private KafkaTemplate<String, Object> kafkaTemplate;
// /**
// * 自定义topicKafkaTemplate
// */
// public static final String TOPIC = "companyBussTest";
//
// /**
// * 自定义topic组
// */
// public static final String TOPIC_GROUP1 = "topic.group1";
//
// public void send(String message) {
//// String message = "test";
// String obj2String = JSONObject.toJSONString(message);
// log.info("准备发送消息为：{}", obj2String);
// //发送消息
// ListenableFuture<SendResult<String, Object>> future = kafkaTemplate.send(TOPIC, message);
// future.addCallback(new ListenableFutureCallback<SendResult<String, Object>>() {
// @Override
// public void onFailure(Throwable throwable) {
// //发送失败的处理
// log.info(TOPIC + " - 生产者 发送消息失败：" + throwable.getMessage());
// }
//
// @Override
// public void onSuccess(SendResult<String, Object> stringObjectSendResult) {
// //成功的处理
// log.info(TOPIC + " - 生产者 发送消息成功：" + stringObjectSendResult.toString());
// }
// });
// }
//}

23
src/main/java/com/bfd/analyze/utils/ThrowMessageUtil.java
View File

@ -0,0 +1,23 @@
package com.bfd.analyze.utils;
import java.io.PrintWriter;
import java.io.StringWriter;
/**
* @author jian.mao
* @date 2023年3月22日
* @description
*/
public class ThrowMessageUtil {
/**
* 获取异常信息
* @param t
* @return
*/
public static String getErrmessage(Throwable t){
StringWriter stringWriter=new StringWriter();
t.printStackTrace(new PrintWriter(stringWriter,true));
return stringWriter.getBuffer().toString();
}
}

136
src/main/resources/application.yml
View File

@ -0,0 +1,136 @@
logging:
level:
root: info
path: ./logs
server:
port: 8000
servlet:
context-path: /analystschedule
tomcat:
uri-encoding: utf-8
max-connections: 20000
max-http-form-post-size: 1
max-threads: 1000
spring:
application:
name: analystschedule
datasource:
url: jdbc:mysql://172.24.12.126:3306/cda_db?serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8&useSSL=true
username: root
password: baifendian123
driver-class-name: com.mysql.cj.jdbc.Driver
kafka:
bootstrap-servers: 172.16.12.55:9092,172.16.12.56:9092,172.16.12.57:9092
producer:
retries: 0
#当有多个消息需要被发送到同一个分区时，生产者会把它们放在同一个批次里。该参数指定了一个批次可以使用的内存大小，按照字节数计算。
batch-size: 16384
# 设置生产者内存缓冲区的大小。
buffer-memory: 33554432
# 键的序列化方式
key-serializer: org.apache.kafka.common.serialization.StringSerializer
# 值的序列化方式
value-serializer: org.apache.kafka.common.serialization.StringSerializer
# acks=0 ： 生产者在成功写入消息之前不会等待任何来自服务器的响应。
# acks=1 ： 只要集群的首领节点收到消息，生产者就会收到一个来自服务器成功响应。
# acks=all ：只有当所有参与复制的节点全部收到消息时，生产者才会收到一个来自服务器的成功响应。
acks: 1
consumer:
# 自动提交的时间间隔 在spring boot 2.X 版本中这里采用的是值的类型为Duration 需要符合特定的格式，如1S,1M,2H,5D
auto-commit-interval: 1S
# 该属性指定了消费者在读取一个没有偏移量的分区或者偏移量无效的情况下该作何处理：
# latest（默认值）在偏移量无效的情况下，消费者将从最新的记录开始读取数据（在消费者启动之后生成的记录）
# earliest ：在偏移量无效的情况下，消费者将从起始位置读取分区的记录
auto-offset-reset: earliest
# 是否自动提交偏移量，默认值是true,为了避免出现重复数据和数据丢失，可以把它设置为false,然后手动提交偏移量
enable-auto-commit: true
# 键的反序列化方式
key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
# 值的反序列化方式
value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
#消费组
group-id: test4
#消费者并发线程数
concurrency: 4
#超时时间
max-poll-interval-ms: 60000
#listener:
# 在侦听器容器中运行的线程数。
#concurrency: 5
#listner负责ack，每调用一次，就立即commit
#ack-mode: manual_immediate
#missing-topics-fatal: false
redis:
host: 172.24.12.126
port: 6379
timeout: 10000
database: 5
jedis:
pool:
max-active: 8 # 连接池最大连接数（使用负值表示没有限制）
max-wait: 800 # 连接池最大阻塞等待时间（使用负值表示没有限制）
max-idle: 8 # 连接池中的最大空闲连接
min-idle: 2 # 连接池中的最小空闲连接
boot:
admin:
client:
#url: http://10.10.143.85:8000
url: http://172.16.12.55:8001
instance:
service-base-url: http://10.10.143.85:8000
management:
endpoints:
web:
exposure:
include: "*"
endpoint:
health:
show-details: always
health:
elasticsearch:
enabled: false
kafka:
consumer:
#自定义topic
topic: analyze
mybatis:
type-aliases-package: com.bfd.analyze.model
elasticsearch:
cluster-nodes: 172.16.12.55:9200,172.16.12.56:9200,172.16.12.57:9200
cluster-name: zh628es-cluster
username: elastic
password: bfd123
index-name: analyze_00001
dispatch:
index-name: dispatch_analyze_00001
zookeeper:
connection-string: 172.16.12.55:2181,172.16.12.56:2181,172.16.12.57:2181
publish-node: /analyze
task:
dispatch-thread-num: 1
results-thread-num: 1
save-thread-num: 1
queue-size: 10000
error-task-path: ../data/errorTask.txt
error-result-path: ../data/errorResult.txt
task-queue-path: ../data/taskQueue.txt
result-queue-path: ../data/resultQueue.txt
saveData-queue-path: ../data/saveDataQueue.txt
errorTask-queue-path: ../data/errorTaskQueue.txt
cycle-intv: 180000 #m默认3分钟
error-time: 3 #应用请求最大重试次数
#计费配置
billing:
apps:
2: 语音识别
3: 文字翻译
1027: 文档翻译
17: 文字识别
65: 生成式大模型
billingUrl: http://172.24.12.126:7076
aimodelType: 65

36
src/main/resources/logback-spring.xml
View File

@ -0,0 +1,36 @@
<configuration>
<!-- 属性文件:在properties文件中找到对应的配置项 -->
<springProperty scope="context" name="logging.path" source="logging.path"/>
<springProperty scope="context" name="logging.level" source="logging.level.com.bfd"/>
<!-- 默认的控制台日志输出，一般生产环境都是后台启动，这个没太大作用 -->
<!-- <appender name="STDOUT"
class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<Pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</Pattern>
</encoder>
</appender> -->
<appender name="GLMAPPER-LOGGERONE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<append>true</append>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>${logging.level}</level>
</filter>
<file>
${logging.path}/analyzeInfoscheduling.log
</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${logging.path}/analyzeInfoscheduling.log.%d{yyyy-MM-dd}</FileNamePattern>
<MaxHistory>7</MaxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<root level="info">
<appender-ref ref="GLMAPPER-LOGGERONE"/>
<!-- <appender-ref ref="STDOUT"/> -->
</root>
</configuration>

20
src/test/java/com/bfd/AppTest.java
View File

@ -0,0 +1,20 @@
package com.bfd;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
/**
* Unit test for simple App.
*/
public class AppTest
{
/**
* Rigorous Test :-)
*/
@Test
public void shouldAnswerWithTrue()
{
assertTrue( true );
}
}
Write Preview
Loading…
Cancel
Save