commit b6c839c638842915beda22486097294f5f368634 Author: 55007 <55007@maojian> Date: Tue Jan 7 16:03:27 2025 +0800 有知数据流 diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..deb4b51 --- /dev/null +++ b/.classpath @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aa25188 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/target/ +/logs/ +/.idea/ \ No newline at end of file diff --git a/.project b/.project new file mode 100644 index 0000000..7dca559 --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + analyst_assistant_schedule + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..365bbd6 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..0ada971 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,9 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.methodParameters=generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..14b697b --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/README.md b/README.md new file mode 100644 index 0000000..7c67fd0 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +分析助手应用调度 diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..2034791 --- /dev/null +++ b/pom.xml @@ -0,0 +1,269 @@ + + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 2.2.4.RELEASE + + com.bfd.analyze + analyst_assistant_schedule + 0.0.1-SNAPSHOT + + analyst_assistant_schedule + + http://www.example.com + + + UTF-8 + 1.8 + 1.8 + + + + + junit + junit + 4.11 + test + + + org.springframework.boot + spring-boot-starter-web + + + + de.codecentric + spring-boot-admin-starter-client + 2.2.4 + + + com.google.code.gson + gson + 2.8.8 + + + org.mybatis.spring.boot + mybatis-spring-boot-starter + 2.0.0 + + + org.springframework.boot + spring-boot-test + + + + org.springframework + spring-test + 5.0.10.RELEASE + test + + + commons-io + commons-io + 1.4 + + + com.alibaba + fastjson + 2.0.17 + + + + com.mchange + c3p0 + 0.9.5.5 + + + mysql + mysql-connector-java + 8.0.29 + + + + com.squareup.okhttp3 + okhttp + 4.9.3 + + + org.apache.httpcomponents + httpclient + 4.5.3 + + + commons-lang + commons-lang + 2.6 + + + + org.jetbrains.kotlin + kotlin-reflect + 1.6.21 + runtime + + + + redis.clients + jedis + 3.3.0 + + + + org.jsoup + jsoup + 1.8.1 + + + + org.projectlombok + lombok + + + + + + + + org.springframework.kafka + spring-kafka + + + cn.hutool + hutool-all + 5.8.5 + + + junit + junit + + + + p6spy + p6spy + 3.9.0 + + + + commons-collections + commons-collections + 3.2.2 + + + + + + org.redisson + redisson-spring-boot-starter + 3.13.6 + + + org.springframework.boot + spring-boot-starter-data-redis + + + org.apache.curator + curator-framework + 5.2.0 + + + org.apache.curator + curator-recipes + 5.2.0 + + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + org.springframework.boot + spring-boot-maven-plugin + + com.bfd.analyze.Application + ZIP + + + ${project.groupId} + ${project.artifactId} + + + + + + + repackage + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.1.1 + + + copy + package + + copy-dependencies + + + jar + jar + runtime + ${project.build.directory}/libs + + + + + + + + diff --git a/src/main/java/com/bfd/analyze/Application.java b/src/main/java/com/bfd/analyze/Application.java new file mode 100644 index 0000000..9efb4ec --- /dev/null +++ b/src/main/java/com/bfd/analyze/Application.java @@ -0,0 +1,29 @@ +package com.bfd.analyze; + + + +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * 主入口 + * + * @author jian.mao + * @date 2023年7月4日 + * @description + */ +@SpringBootApplication +@MapperScan("com.bfd.analyze.mapper") +@EnableScheduling +@EnableKafka +public class Application { + + public static void main(String[] args) { + SpringApplication.run(Application.class, args); + } + + +} diff --git a/src/main/java/com/bfd/analyze/cache/ConfigCache.java b/src/main/java/com/bfd/analyze/cache/ConfigCache.java new file mode 100644 index 0000000..3a64191 --- /dev/null +++ b/src/main/java/com/bfd/analyze/cache/ConfigCache.java @@ -0,0 +1,49 @@ +package com.bfd.analyze.cache; + +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.concurrent.LinkedBlockingDeque; + +/** + * @author jian.mao + * @date 2022年11月11日 + * @description 静态变量类 + */ +@Slf4j +public class ConfigCache { + + /**启动条件**/ + public static boolean isStart = true; + /*****任务队列*****/ + public static LinkedBlockingDeque> taskQueue = null; + /*****结果集队列*****/ + public static LinkedBlockingDeque> ResultQueue = null; + /*****持久化到es队列*****/ + public static LinkedBlockingDeque> saveDataQueue = null; + /*****错误任务队列*****/ + public static LinkedBlockingDeque> errorTaskQueue = null; + /*****周期任务等待队列*****/ + public static LinkedBlockingDeque> cycleTaskWaitQueue = null; + + /*****等待app任务缓存****/ + /***** + public static Map waitTask = new HashMap(1024); + * ****/ + + public static String resultTopic = null; + + /** + * 队列录入任务 + * @param queue + * @param task + */ + public static void putQueue(LinkedBlockingDeque> queue,Map task){ + //next app 写入队列准备调出 + try { + queue.put(task); + } catch (InterruptedException e) { + log.error("队列写入data失败---"); + } + } +} diff --git a/src/main/java/com/bfd/analyze/config/BillingConfig.java b/src/main/java/com/bfd/analyze/config/BillingConfig.java new file mode 100644 index 0000000..9a851ec --- /dev/null +++ b/src/main/java/com/bfd/analyze/config/BillingConfig.java @@ -0,0 +1,42 @@ +package com.bfd.analyze.config; + +import java.util.Map; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * @author jian.mao + * @date 2024年9月24日 + * @description + */ +@Component +@ConfigurationProperties(prefix = "billing") +public class BillingConfig { + + private Map apps; + private String billingUrl; + private int aimodelType; + + + + public Map getApps() { + return apps; + } + public void setApps(Map apps) { + this.apps = apps; + } + public String getBillingUrl() { + return billingUrl; + } + public void setBillingUrl(String billingUrl) { + this.billingUrl = billingUrl; + } + public int getAimodelType() { + return aimodelType; + } + public void setAimodelType(int aimodelType) { + this.aimodelType = aimodelType; + } + +} diff --git a/src/main/java/com/bfd/analyze/config/KafkaConfig.java b/src/main/java/com/bfd/analyze/config/KafkaConfig.java new file mode 100644 index 0000000..78e5b7d --- /dev/null +++ b/src/main/java/com/bfd/analyze/config/KafkaConfig.java @@ -0,0 +1,20 @@ +package com.bfd.analyze.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +/** + * kafka配置类 + * @author jian.mao + * @date 2023年7月6日 + * @description + */ +@Configuration +public class KafkaConfig { + @Value("${kafka.consumer.topic}") + private String kafkaTopic; + + public String getKafkaTopic() { + return kafkaTopic; + } +} diff --git a/src/main/java/com/bfd/analyze/config/ThreadPoolConfig.java b/src/main/java/com/bfd/analyze/config/ThreadPoolConfig.java new file mode 100644 index 0000000..4e69f2f --- /dev/null +++ b/src/main/java/com/bfd/analyze/config/ThreadPoolConfig.java @@ -0,0 +1,43 @@ +package com.bfd.analyze.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.*; + +/** + * @author jian.mao + * @date 2023年3月23日 + * @description + */ +@Configuration +public class ThreadPoolConfig { + /** 核心线程数（默认线程数） */ + private static final int CORE_POOL_SIZE = 20; + /** 最大线程数 */ + private static final int MAX_POOL_SIZE = 50; + /** 允许线程空闲时间（单位：默认为秒） */ + private static final int KEEP_ALIVE_TIME = 10; + /** 缓冲队列大小 */ + private static final int QUEUE_CAPACITY = 200; + /** 线程池名前缀 */ + private static final String THREAD_NAME_PREFIX = "ThreadPool-"; + + @Bean("taskExecutor") + public ThreadPoolTaskExecutor taskExecutor(){ + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(CORE_POOL_SIZE); + executor.setMaxPoolSize(MAX_POOL_SIZE); + executor.setQueueCapacity(QUEUE_CAPACITY); + executor.setKeepAliveSeconds(KEEP_ALIVE_TIME); + executor.setThreadNamePrefix(THREAD_NAME_PREFIX); + + // 线程池对拒绝任务的处理策略 + // CallerRunsPolicy：由调用线程（提交任务的线程）处理该任务 + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); + // 初始化 + executor.initialize(); + return executor; + } +} diff --git a/src/main/java/com/bfd/analyze/config/ZookeeperConfig.java b/src/main/java/com/bfd/analyze/config/ZookeeperConfig.java new file mode 100644 index 0000000..25d9896 --- /dev/null +++ b/src/main/java/com/bfd/analyze/config/ZookeeperConfig.java @@ -0,0 +1,25 @@ +package com.bfd.analyze.config; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author jian.mao + * @date 2024年4月16日 + * @description + */ +@Configuration +public class ZookeeperConfig { + @Value("${zookeeper.connection-string}") + private String connectionString; + @Bean + public CuratorFramework curatorFramework() { + CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(connectionString, new ExponentialBackoffRetry(1000, 3)); + curatorFramework.start(); + return curatorFramework; + } +} diff --git a/src/main/java/com/bfd/analyze/controller/ProcessTriggerController.java b/src/main/java/com/bfd/analyze/controller/ProcessTriggerController.java new file mode 100644 index 0000000..3c87e31 --- /dev/null +++ b/src/main/java/com/bfd/analyze/controller/ProcessTriggerController.java @@ -0,0 +1,115 @@ +package com.bfd.analyze.controller; + +import javax.annotation.Resource; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.curator.framework.CuratorFramework; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; + +import com.bfd.analyze.service.ProcessTriggerService; + +/** + * 任务管理接口 + * + * @author jian.mao + * @date 2023年9月19日 + * @description + */ +@Controller +@RequestMapping("/trigger") +@Slf4j +public class ProcessTriggerController { + + @Autowired + private CuratorFramework curatorFramework; + @Resource + private ProcessTriggerService processTriggerService; + + @PostMapping("/start") + @ResponseBody + public String processStart(@RequestBody String dataJson) { + String result = processTriggerService.start(dataJson); + return result; + } + + @RequestMapping(value = "/hello", method = RequestMethod.GET) + @ResponseBody + public String hello(String param, String token) { + return "123"; + } + + @PostMapping("/stop") + @ResponseBody + public String processStop(@RequestBody String dataJson) { + String result = processTriggerService.stop(dataJson); + return result; + } + + /** + * 场景删除 + * @param dataJson + * @return + */ + @PostMapping("/scenesDelete") + @ResponseBody + public String scenesDelete(@RequestBody String dataJson) { + String result = processTriggerService.scenesDelete(dataJson); + return result; + } + + + /** + * 流程主动触发 + * @param dataJson + * @return + */ + @PostMapping("/active") + @ResponseBody + public String activeExec(@RequestBody String dataJson) { + String result = processTriggerService.active(dataJson); + return result; + } + + /** + * 暂停 + * @param dataJson + * @return + */ + @PostMapping("/pause") + @ResponseBody + public String pause(@RequestBody String dataJson) { + String result = processTriggerService.pause(dataJson); + return result; + } + + /** + * 逻辑删除数据 + * @param dataJson + * @return + */ + @PostMapping("/delete") + @ResponseBody + public String delete(@RequestBody String dataJson) { + String result = processTriggerService.delete(dataJson); + return result; + } + + /** + * 重试 + * @param dataJson + * @return + */ + @PostMapping("/retry") + @ResponseBody + public String retry(@RequestBody String dataJson) { + String result = processTriggerService.retry(dataJson); + return result; + } +} diff --git a/src/main/java/com/bfd/analyze/entity/AnalystData.java b/src/main/java/com/bfd/analyze/entity/AnalystData.java new file mode 100644 index 0000000..8338260 --- /dev/null +++ b/src/main/java/com/bfd/analyze/entity/AnalystData.java @@ -0,0 +1,41 @@ +package com.bfd.analyze.entity; + +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * 最终存储的数据体实体类 + * + * @author jian.mao + * @date 2023年7月11日 + * @description + */ +@Data +public class AnalystData { + + private String created; + private String lastEdit; + private Integer appId; + private Integer transferId; + private String appName; + private String appCode; + private String appDescribe; + private String module; + private String result; + private List> nextAppId; + private Boolean startTag; + private Integer scenesId; + private String businessKey; + private String dataId; + private String dataProcessId; + private List> variable; + private Integer del; + private Integer status; + private String message; + private Integer version; + private String dispatchId; + private List parentId; + private Integer trace; +} diff --git a/src/main/java/com/bfd/analyze/entity/Constants.java b/src/main/java/com/bfd/analyze/entity/Constants.java new file mode 100644 index 0000000..5623e45 --- /dev/null +++ b/src/main/java/com/bfd/analyze/entity/Constants.java @@ -0,0 +1,149 @@ +package com.bfd.analyze.entity; + + +/** + * 常量实体类 + * @author jian.mao + * @date 2022年11月15日 + * @description + */ +public class Constants { + /*************************蓝图常量key名称*********************************/ + public final static String SCHEDULING = "scheduling"; + public final static String TYPE = "type"; + public final static String INTERVAL = "interval"; + public final static String CREATED = "created"; + public final static String LAST_EDIT = "last_edit"; + public final static String BLUEPRINT_ID = "blueprint_id"; + public final static String BLUEPRINT_NAME = "name"; + public final static String SCENARIO = "scenario"; + public final static String AUTOCOMMITTRIGGERLAST = "autoCommitTriggerLast"; + public final static String FRESHVARIABLES = "freshVariables"; + public final static String AUTOCOMMIT = "autoCommit"; + public final static String MAXERRORS = "maxErrors"; + public final static String DATALOSS = "dataloss"; + public final static String POSITION = "position"; + public final static String SCENES_ID = "scenes_id"; + public final static String COVER = "cover"; + public final static String MULTI_BRANCH = "multi_branch"; + + public final static String SINGLE = "single"; + /**已重试次数**/ + public final static String ERROR_TIME = "error_time"; + public final static String PREVIOUS_RESULT = "previous_result"; + + /****数据id*****/ + public final static String BUSINESSKEY = "businessKey"; + + /*************************metadata常量key名称*********************************/ + public final static String OUTPUT = "output"; + public final static String LABEL_COL = "label_col"; + public final static String LABEL = "label"; + public final static String INPUT = "input"; + public final static String USER = "user"; + public final static String ADMIN = "admin"; + public final static String ADDRESS = "address"; + public final static String DATASOURCE = "datasource"; + public final static String INDEX = "index"; + + /*************************app常量key名称*********************************/ + public final static String APPS = "apps"; + public final static String APP_ID = "id"; + public final static String TRANSFER_ID = "transfer_id"; + public final static String MODULE = "module"; + public final static String VERSION = "version"; + public final static String DATA = "data"; + public final static String APP_NAME = "name"; + public final static String DESCRIBE = "describe"; + public final static String APP_CODE = "app_code"; + public final static String NEXT_APP_ID = "next_app_id"; + public final static String EDGE_ID = "edge_id"; + public final static String START_ID = "start_id"; + public final static String END_ID = "end_id"; + public static final String DATAID = "dataId"; + public static final String NOTE = "note"; + + + public final static String WAIT_CONDITION = "wait_condition"; + public final static String START_TAG = "start_tag"; + public final static String RELATIONS = "relations"; + public static final String SCHEDULING_TYPE = "scheduling_type"; + public static final String NEXT_SCHEDULING_TIME = "next_scheduling_time"; + public static final String FLAG_DATA = "flag_data"; + /*************************module类型*********************************/ + public final static String FILE = "file"; + public final static String OCR = "OCR"; + public final static String FILTER = "Filter"; + public final static String CHATGPT = "ChatGPT"; + public final static String MYSQL = "mysql"; + + /*************************other类型*********************************/ + public final static String UNDERLINE = "_"; + public final static String RESULT_TOPIC = null; + public static final String EMPTY = ""; + public static final String HTTP = "http"; + public static final String REQUEST_ERROR_MESSAGE = "Download failed error is"; + public static final String REQUEST_RESULT = "result"; + public static final String REQUEST_RESULT_RESULTS = "results"; + public static final String MAP_TYPE = "Map"; + public static final String LIST_TYPE = "List"; + public static final String STRING_TYPE = "String"; + public static final String DOCUMENT_TYPE = "doc"; + public static final String FILTER_ZH = "过滤器"; + + public static final String JSON_SELE_SYMBOL = "$."; + public static final String LEFT_BRACKETS = "["; + public static final String RIGTH_BRACKETS = "]"; + public static final String ISLAST = "isLast"; + /****是否是扩散型应用标识key*****/ + public static final String IS_DIFFUSION = "is_diffusion"; + /************************redis*************************************/ + public static final String LOCK_KEY = "myLock"; + public static final long LOCK_EXPIRE_TIME = 300000; + public static final String CODE = "code"; + public static final String MESSAGE = "message"; + public static final String FORM = "form"; + public static final String FIELD = "field"; + public static final String SOURCE_DATA_ID = "source_data_id"; + public static final String WILDCARD = "*"; + public static final String DATAPROCESSID = "dataProcessId"; + public static final String HITS = "hits"; + public static final String ES_SOURCE = "_source"; + public static final String APPCODE = "appCode"; + public static final String CREATEUSERID = "createUserId"; + public static final long LOCKTIME = 10L; + public static final String IS_ACTIVE = "is_active"; + public static final String DATAIDS = "dataIds"; + public static final String SEARCHFIELDS = "searchFields"; + public static final String DEL = "del"; + public static final String UPDATED = "updated"; + public static final String STATUS = "status"; + public static final String NEXTAPPID = "nextAppId"; + public static final String SCENESID = "scenesId"; + /**聚合类应用，多个数据到此应用只产生一条结果**/ + public static final String POLYMERIZATION = "polymerization"; + /*************************接口动作************************************/ + public static final String START = "start"; + + public static final String STOP = "stop"; + public static final String DELETE = "delete"; + public static final String MATCH = "match"; + public static final String TERMS = "terms"; + public static final String TERM = "term"; + public static final String DISPATCHID = "dispatchId"; + public static final String PARENTID = "parentId"; + public static final String POINT_KEYWORD = ".keyword"; + public static final String DEFAULTCOMPLETION = "defaultCompletion"; + public static final String ERROR = "error"; + public static final String PLANCODE = "planCode"; + public static final String SCENES_NAME = "scenes_name"; + + public static final String CDA_ = "cda_"; + public static final String MODULE_ID = "module_id"; + public static final String MODELTYPE = "modelType"; + public static final String BILLINGSUFFIX = "/api/pay/api/publicTransaction/spend"; + public static final String TRACE = "trace"; + + + +} diff --git a/src/main/java/com/bfd/analyze/handler/DataExecHandler.java b/src/main/java/com/bfd/analyze/handler/DataExecHandler.java new file mode 100644 index 0000000..a264253 --- /dev/null +++ b/src/main/java/com/bfd/analyze/handler/DataExecHandler.java @@ -0,0 +1,314 @@ +package com.bfd.analyze.handler; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.analyze.cache.ConfigCache; +import com.bfd.analyze.entity.Constants; +import com.bfd.analyze.mapper.AppsMapper; +import com.bfd.analyze.mapper.BlueprintMapper; +import com.bfd.analyze.mapper.ModulesMapper; +import com.bfd.analyze.mapper.RelationsMapper; +import com.bfd.analyze.process.ErrorTaskProcess; +import com.bfd.analyze.process.ResultParseProcess; +import com.bfd.analyze.process.SaveDataProcess; +import com.bfd.analyze.process.TaskdispatchProcess; +import com.bfd.analyze.utils.DateUtil; +import com.bfd.analyze.utils.FileUtil; +import com.bfd.analyze.utils.PauseTool; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.curator.framework.CuratorFramework; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.core.annotation.Order; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingDeque; + + +/** + * 调度执行入口 + * + * @author jian.mao + * @date 2023年7月4日 + * @description + */ +@Component +@Order(value = 1) +@Slf4j +public class DataExecHandler implements ApplicationRunner { + @Resource + private BlueprintMapper blueprintMapper; + @Resource + private AppsMapper appsMapper; + @Resource + private ModulesMapper modulesMapper; + @Resource + private RelationsMapper relationsMapper; + @Autowired + private TaskdispatchProcess taskdispatchProcess; + @Autowired + private ResultParseProcess resultParseProcess; + @Autowired + private SaveDataProcess saveDataProcess; + @Autowired + private ErrorTaskProcess errorTaskProcess; + + @Value("${task.task-queue-path}") + private String taskPath; + + @Value("${task.result-queue-path}") + private String resultPath; + + @Value("${task.saveData-queue-path}") + private String saveDataPath; + + @Value("${task.errorTask-queue-path}") + private String errorTaskPath; + + @Value("${task.dispatch-thread-num}") + private Integer dispatchThreadNum; + + @Value("${task.results-thread-num}") + private Integer resultsThreadNum; + + @Value("${task.save-thread-num}") + private Integer saveThreadNum; + + @Value("${task.queue-size}") + private Integer queueSize; + + @Autowired + private StringRedisTemplate stringRedisTemplate; + @Autowired + private CuratorFramework curatorFramework; + @Autowired + PauseTool pauseTool; + + /** + @Autowired + private DistributedLockService distributedLockService; + + @Autowired + private StringRedisTemplate stringRedisTemplate; + **/ + + +/********************************************************start***************************************************************/ + /** + * 执行入口 + */ + @Override + public void run(ApplicationArguments args) { + //设置队列长度 + if(queueSize == null || queueSize == 0){ + queueSize = 10000; + } + /*****任务队列*****/ + ConfigCache.taskQueue = new LinkedBlockingDeque>(queueSize); + /*****结果集队列*****/ + ConfigCache.ResultQueue = new LinkedBlockingDeque>(queueSize); + /*****持久化到es队列*****/ + ConfigCache.saveDataQueue = new LinkedBlockingDeque>(queueSize); + /*****错误任务队列*****/ + ConfigCache.errorTaskQueue = new LinkedBlockingDeque>(); + /*****周期任务等待队列*****/ + ConfigCache.cycleTaskWaitQueue = new LinkedBlockingDeque>(); + //等待es连接成功在注入es连接对象 + DateUtil.sleep(5000); + if(dispatchThreadNum == null || dispatchThreadNum == 0){ + dispatchThreadNum = 3; + } + if(resultsThreadNum == null || resultsThreadNum == 0){ + resultsThreadNum = 3; + } + if(saveThreadNum == null || saveThreadNum == 0){ + saveThreadNum = 3; + } + for (int i = 0; i < dispatchThreadNum; i++) { + log.info("启动发送线程---"); + new Thread(taskdispatchProcess).start(); + } + for (int i = 0; i < resultsThreadNum; i++) { + log.info("启动调回线程---"); + new Thread(resultParseProcess).start(); + } + for (int i = 0; i < saveThreadNum; i++) { + log.info("启动数据保存线程---"); + new Thread(saveDataProcess).start(); + } + log.info("启动错误任务处理线程---"); + new Thread(errorTaskProcess).start(); + //启动加载缓存任务 + readTask(taskPath,ConfigCache.taskQueue); + readTask(resultPath,ConfigCache.ResultQueue); + readTask(saveDataPath,ConfigCache.saveDataQueue); + readTask(errorTaskPath,ConfigCache.errorTaskQueue); + + //初始化 redis 场景版本缓存 + pauseTool.initializeRedisCache(stringRedisTemplate); + + //停止处理 + waitDown(); + } + + public static void readTask(String path,LinkedBlockingDeque> queue){ + File file = new File(path); + if(file.exists()){ + List tasks = null; + try { + tasks = FileUtils.readLines(file,"UTF-8"); + } catch (IOException e) { + e.printStackTrace(); + } + for (String taskStr : tasks) { + Map task = JSONObject.parseObject(taskStr); + try { + queue.put(task); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + file.delete(); + } + } + + /** + * 查询表，组装任务 + */ + @Scheduled(cron = "*/10 * * * * ?") + private void scanTask() { + /**************************************************old start********************************************************/ + //获取分布式锁 +// if(distributedLockService.acquireLock()){ +// try { +// //打印队列长度 +// log.info("发送队列长度：{}",ConfigCache.taskQueue.size()); +// log.info("调回队列长度：{}",ConfigCache.ResultQueue.size()); +// log.info("保存队列长度：{}",ConfigCache.saveDataQueue.size()); +// log.info("错误队列长度：{}",ConfigCache.errorTaskQueue.size()); +// //redis分布式锁测试 +// log.info("我已占有分布式锁------------------------------------------{}",distributedLockService.acquireLock()); +// +// +// } catch (Throwable e) { +// log.error("扫描任务未知错误----",e); +// }finally{ +// //释放锁 +// distributedLockService.releaseLock(); +// } +// }else{ +// log.info("任务扫描锁未抢占成功------"); +// } + /**************************************************old end********************************************************/ + try { + //打印队列长度 + log.info("发送队列长度：{}", ConfigCache.taskQueue.size()); + log.info("调回队列长度：{}", ConfigCache.ResultQueue.size()); + log.info("保存队列长度：{}", ConfigCache.saveDataQueue.size()); + log.info("错误队列长度：{}", ConfigCache.errorTaskQueue.size()); + log.info("周期等待任务队列长度：{}", ConfigCache.cycleTaskWaitQueue.size()); + while (ConfigCache.cycleTaskWaitQueue.size() > 0) { + Map task = ConfigCache.cycleTaskWaitQueue.poll(); + long currentTimeMillis = System.currentTimeMillis(); + long nextSchedulingTime = (long) task.get(Constants.NEXT_SCHEDULING_TIME); + if (currentTimeMillis > nextSchedulingTime) { + //符合调出 + ConfigCache.taskQueue.put(task); + log.info("周期任务调出：{}", JSONObject.toJSONString(task)); + } else { + //不符合调出时间，放回队列，进行等待 + ConfigCache.cycleTaskWaitQueue.put(task); + } + } + } catch (Throwable e) { + log.error("扫描任务未知错误----", e); + } + } + + + +/**************************************************stop***********************************************************************/ + /** + * 结束触发钩子 + */ + public void waitDown() { + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + // 停止线程 + ConfigCache.isStart = false; + //释放分布式锁 +// distributedLockService.releaseLock(); + log.info("stop-------"); + writeTsskToFile(); + } + }); + } + + /** + * 任务持久化到硬盘 + */ + public void writeTsskToFile(){ + while(true){ + if(ConfigCache.taskQueue.size() > 0 ){ + try { + Map task = ConfigCache.taskQueue.take(); + FileUtil.writeFile(taskPath, JSONObject.toJSONString(task)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }else{ + log.info("taskQueue write is file end"); + break; + } + } + while(true){ + if(ConfigCache.ResultQueue.size() > 0 ){ + try { + Map task = ConfigCache.ResultQueue.take(); + FileUtil.writeFile(resultPath, JSONObject.toJSONString(task)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }else{ + log.info("ResultQueue write is file end"); + break; + } + } + while(true){ + if(ConfigCache.saveDataQueue.size() > 0 ){ + try { + Map task = ConfigCache.saveDataQueue.take(); + FileUtil.writeFile(saveDataPath, JSONObject.toJSONString(task)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }else{ + log.info("saveDataQueue write is file end"); + break; + } + } + while(true){ + if(ConfigCache.errorTaskQueue.size() > 0 ){ + try { + Map task = ConfigCache.errorTaskQueue.take(); + FileUtil.writeFile(errorTaskPath, JSONObject.toJSONString(task)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }else{ + log.info("errorTaskQueue write is file end"); + break; + } + } + } +} \ No newline at end of file diff --git a/src/main/java/com/bfd/analyze/mapper/AppsMapper.java b/src/main/java/com/bfd/analyze/mapper/AppsMapper.java new file mode 100644 index 0000000..f6f594d --- /dev/null +++ b/src/main/java/com/bfd/analyze/mapper/AppsMapper.java @@ -0,0 +1,71 @@ +package com.bfd.analyze.mapper; + +import com.bfd.analyze.model.AppsEntity; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Result; +import org.apache.ibatis.annotations.Results; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + + +/** + * APP节点mapper接口 + * @author jian.mao + * @date 2023年7月5日 + * @description + */ +public interface AppsMapper { + + @Select({"select * from apps where scenes_id = #{scenesId} and del = 0"}) + @Results({ + @Result(column = "id",property = "id"), + @Result(column = "create_user_id",property = "createUserId"), + @Result(column = "create_user",property = "createUser"), + @Result(column = "update_user_id",property = "updateUserId"), + @Result(column = "update_user",property = "updateUser"), + @Result(column = "create_time",property = "createTime"), + @Result(column = "update_time",property = "updateTime"), + @Result(column = "del",property = "del"), + @Result(column = "app_code",property = "appCode"), + @Result(column = "transfer_id",property = "transferId"), + @Result(column = "scenes_id",property = "scenesId"), + @Result(column = "module_id",property = "moduleId"), + @Result(column = "name",property = "name"), + @Result(column = "describe",property = "describe"), + @Result(column = "input",property = "input"), + @Result(column = "output",property = "output"), + @Result(column = "flag_app",property = "flagApp"), + @Result(column = "flag_data",property = "flagData"), + @Result(column = "scheduling_type",property = "schedulingType"), + @Result(column = "wait_condition",property = "waitCondition") + }) + public List getAppByscenesId(@Param("scenesId") Integer scenesId); + + + + @Select({"select * from apps where app_code = #{appCode}"}) + @Results({ + @Result(column = "id",property = "id"), + @Result(column = "create_user_id",property = "createUserId"), + @Result(column = "create_user",property = "createUser"), + @Result(column = "update_user_id",property = "updateUserId"), + @Result(column = "update_user",property = "updateUser"), + @Result(column = "create_time",property = "createTime"), + @Result(column = "update_time",property = "updateTime"), + @Result(column = "del",property = "del"), + @Result(column = "app_code",property = "appCode"), + @Result(column = "transfer_id",property = "transferId"), + @Result(column = "scenes_id",property = "scenesId"), + @Result(column = "module_id",property = "moduleId"), + @Result(column = "name",property = "name"), + @Result(column = "describe",property = "describe"), + @Result(column = "input",property = "input"), + @Result(column = "output",property = "output"), + @Result(column = "flag_app",property = "flagApp"), + @Result(column = "flag_data",property = "flagData"), + @Result(column = "scheduling_type",property = "schedulingType"), + @Result(column = "wait_condition",property = "waitCondition") + }) + public AppsEntity getAppByAppCode(@Param("appCode") String appCode); +} diff --git a/src/main/java/com/bfd/analyze/mapper/BlueprintMapper.java b/src/main/java/com/bfd/analyze/mapper/BlueprintMapper.java new file mode 100644 index 0000000..c56dcf7 --- /dev/null +++ b/src/main/java/com/bfd/analyze/mapper/BlueprintMapper.java @@ -0,0 +1,51 @@ +package com.bfd.analyze.mapper; + +import java.util.List; + +import org.apache.ibatis.annotations.Result; +import org.apache.ibatis.annotations.Results; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +import com.bfd.analyze.model.BlueprintEntity; + + +/** + * 蓝图mapper接口 + * @author jian.mao + * @date 2023年7月5日 + * @description + */ +public interface BlueprintMapper { + + @Select({"select * from blueprint where status = 1 and (next_scheduling_time is null or next_scheduling_time < now())"}) + @Results({ + @Result(column = "blueprint_id",property = "blueprintId"), + @Result(column = "scenes_id",property = "scenesId"), + @Result(column = "name",property = "name"), + @Result(column = "scheduling_type",property = "schedulingType"), + @Result(column = "scheduling_interval",property = "schedulingInterval"), + @Result(column = "auto_commit_trigger_last",property = "autoCommitTriggerLast"), + @Result(column = "dataloss",property = "dataloss"), + @Result(column = "max_errors",property = "maxErrors"), + @Result(column = "auto_commit",property = "autoCommit"), + @Result(column = "fresh_variables",property = "freshVariables"), + @Result(column = "created",property = "created"), + @Result(column = "last_edit",property = "lastEdit"), + @Result(column = "user",property = "user"), + @Result(column = "multi_branch",property = "multiBranch"), + }) + public List getAll(); + + /** + * 单次任务 + * @param blueprintEntity + */ + @Update("update blueprint set status = 3 where blueprint_id = #{blueprintId}") + public void updateSingleBlueprintById(BlueprintEntity blueprintEntity); + + @Update("update blueprint set next_scheduling_time = date_add(now(),interval #{schedulingInterval} second) where blueprint_id = #{blueprintId}") + public void updateCycleBlueprintById(BlueprintEntity blueprintEntity); + + +} diff --git a/src/main/java/com/bfd/analyze/mapper/ModulesMapper.java b/src/main/java/com/bfd/analyze/mapper/ModulesMapper.java new file mode 100644 index 0000000..612ba90 --- /dev/null +++ b/src/main/java/com/bfd/analyze/mapper/ModulesMapper.java @@ -0,0 +1,42 @@ +package com.bfd.analyze.mapper; + + +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Result; +import org.apache.ibatis.annotations.Results; +import org.apache.ibatis.annotations.Select; + +import com.bfd.analyze.model.ModulesEntity; + + +/** + * APP节点mapper接口 + * @author jian.mao + * @date 2023年7月5日 + * @description + */ +public interface ModulesMapper { + + @Select({"select * from modules where id = #{moduleId}"}) + @Results({ + @Result(column = "id",property = "id"), + @Result(column = "create_user_id",property = "createUserId"), + @Result(column = "create_user",property = "createUser"), + @Result(column = "update_user_id",property = "updateUserId"), + @Result(column = "update_user",property = "updateUser"), + @Result(column = "create_time",property = "createTime"), + @Result(column = "update_time",property = "updateTime"), + @Result(column = "del",property = "del"), + @Result(column = "name",property = "name"), + @Result(column = "describe",property = "describe"), + @Result(column = "logo",property = "logo"), + @Result(column = "category_id",property = "categoryId"), + @Result(column = "params",property = "params"), + @Result(column = "output",property = "output"), + @Result(column = "output_example",property = "outputExample"), + @Result(column = "flag_release",property = "flagRelease"), + @Result(column = "flag_start",property = "flagStart"), + @Result(column = "url_start",property = "urlStart") + }) + public ModulesEntity getModuleByModuleId(@Param("moduleId") Integer moduleId); +} diff --git a/src/main/java/com/bfd/analyze/mapper/RelationsMapper.java b/src/main/java/com/bfd/analyze/mapper/RelationsMapper.java new file mode 100644 index 0000000..11cfd3f --- /dev/null +++ b/src/main/java/com/bfd/analyze/mapper/RelationsMapper.java @@ -0,0 +1,37 @@ +package com.bfd.analyze.mapper; + +import com.bfd.analyze.model.RelationsEntity; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Result; +import org.apache.ibatis.annotations.Results; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + + +/** + * APP节点mapper接口 + * @author jian.mao + * @date 2023年7月5日 + * @description + */ +public interface RelationsMapper { + + @Select({"select * from relations where scenes_id = #{scenesId} and del = 0"}) + @Results({ + @Result(column = "id",property = "id"), + @Result(column = "create_user_id",property = "createUserId"), + @Result(column = "create_user",property = "createUser"), + @Result(column = "update_user_id",property = "updateUserId"), + @Result(column = "update_user",property = "updateUser"), + @Result(column = "create_time",property = "createTime"), + @Result(column = "update_time",property = "updateTime"), + @Result(column = "del",property = "del"), + @Result(column = "scenes_id",property = "scenesId"), + @Result(column = "start_id",property = "startId"), + @Result(column = "end_id",property = "endId"), + @Result(column = "start_code",property = "startCode"), + @Result(column = "end_code",property = "endCode") + }) + public List getRelationsByscenesId(@Param("scenesId") Integer scenesId); +} diff --git a/src/main/java/com/bfd/analyze/mapper/ScenesMapper.java b/src/main/java/com/bfd/analyze/mapper/ScenesMapper.java new file mode 100644 index 0000000..cff8b43 --- /dev/null +++ b/src/main/java/com/bfd/analyze/mapper/ScenesMapper.java @@ -0,0 +1,47 @@ +package com.bfd.analyze.mapper; + +import java.util.List; + +import org.apache.ibatis.annotations.Result; +import org.apache.ibatis.annotations.Results; +import org.apache.ibatis.annotations.Select; + +import com.bfd.analyze.model.ScenesEntity; + + +/** + * 蓝图mapper接口 + * @author jian.mao + * @date 2023年7月5日 + * @description + */ +public interface ScenesMapper { + + @Select({"select * from scenes where id = #{scenesId}"}) + @Results({ + @Result(column = "id",property = "id"), + @Result(column = "create_user_id",property = "createUserId"), + @Result(column = "create_user",property = "createUser"), + @Result(column = "update_user_id",property = "updateUserId"), + @Result(column = "update_user",property = "updateUser"), + @Result(column = "create_time",property = "createTime"), + @Result(column = "update_time",property = "updateTime"), + @Result(column = "del",property = "del"), + @Result(column = "scenes_name",property = "scenesName"), + @Result(column = "scenes_describe",property = "scenesDescribe"), + @Result(column = "version",property = "version"), + @Result(column = "category_id",property = "categoryId"), + @Result(column = "flag_release",property = "flagRelease"), + @Result(column = "flag_owner",property = "flagOwner"), + @Result(column = "status",property = "status"), + @Result(column = "diagram",property = "diagram"), + @Result(column = "original_scenes_id",property = "originalScenesId"), + @Result(column = "source_id",property = "sourceId"), + @Result(column = "source_data_id",property = "sourceDataId"), + @Result(column = "multi_branch",property = "multiBranch"), + @Result(column = "flag_data",property = "flagData"), + + }) + public ScenesEntity getscenesById(Integer scenesId); + +} diff --git a/src/main/java/com/bfd/analyze/model/AppsEntity.java b/src/main/java/com/bfd/analyze/model/AppsEntity.java new file mode 100644 index 0000000..bbd3f3a --- /dev/null +++ b/src/main/java/com/bfd/analyze/model/AppsEntity.java @@ -0,0 +1,56 @@ +package com.bfd.analyze.model; + +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * app节点实体类 + * + * @author jian.mao + * @date 2023年7月5日 + * @description + */ +@Data +public class AppsEntity { + /****主键****/ + private Integer id; + /****创建人id****/ + private String createUserId; + /****创建人姓名****/ + private String createUser; + /****更新人id****/ + private String updateUserId; + /****更新人姓名****/ + private String updateUser; + /****创建时间****/ + private LocalDateTime createTime; + /****更新时间****/ + private LocalDateTime updateTime; + /****删除标识0-未删除，1-已删除****/ + private Integer del; + /*** 流程编码前端生成uuid，输出结果与输入配置均使用此项 ***/ + private String appCode; + /*** 流转id ***/ + private Integer transferId; + /*** 场景id ***/ + private Integer scenesId; + /*** 模块id对应定制化应用 ***/ + private Integer moduleId; + /*** 节点操作名称 ***/ + private String name; + /*** 节点描述 ***/ + private String describe; + /*** 输出 ***/ + private String output; + /*** 输入 ***/ + private String input; + /*** 应用可见0-否 1-是 ***/ + private Integer flagApp; + /*** 数据可见0-否 1-是 ***/ + private Integer flagData; + /****调出模式****/ + private Integer schedulingType; + /****等待节点****/ + private String waitCondition; +} diff --git a/src/main/java/com/bfd/analyze/model/BlueprintEntity.java b/src/main/java/com/bfd/analyze/model/BlueprintEntity.java new file mode 100644 index 0000000..361c022 --- /dev/null +++ b/src/main/java/com/bfd/analyze/model/BlueprintEntity.java @@ -0,0 +1,44 @@ +package com.bfd.analyze.model; + +import java.sql.Timestamp; +import java.time.LocalDateTime; + +import lombok.Data; + +/** + * 蓝图实体类 + * @author jian.mao + * @date 2023年7月5日 + * @description + */ +@Data +public class BlueprintEntity { + /***蓝图id***/ + private Integer blueprintId; + /***场景id***/ + private Integer scenesId; + /***蓝图名称***/ + private String name; + /***调度类型***/ + private String schedulingType; + /***周期***/ + private Integer schedulingInterval; + /***蓝图是否最后提交***/ + private Integer autoCommitTriggerLast; + /***蓝图状态，成功\失败***/ + private Integer dataloss; + /***重试次数***/ + private Integer maxErrors; + /***自动提交***/ + private Integer autoCommit; + /*** ***/ + private Integer freshVariables; + /***创建时间***/ + private LocalDateTime created; + /***最后修改时间***/ + private Timestamp lastEdit; + /*** ***/ + private String user; + /***单、多分支标识***/ + public Integer multiBranch; +} diff --git a/src/main/java/com/bfd/analyze/model/ModulesEntity.java b/src/main/java/com/bfd/analyze/model/ModulesEntity.java new file mode 100644 index 0000000..2515990 --- /dev/null +++ b/src/main/java/com/bfd/analyze/model/ModulesEntity.java @@ -0,0 +1,53 @@ +package com.bfd.analyze.model; + +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * 模块实体类 + * + * @author jian.mao + * @date 2023年7月5日 + * @description + */ +@Data +public class ModulesEntity { + /****主键****/ + private Integer id; + /****创建人id****/ + private String createUserId; + /****创建人姓名****/ + private String createUser; + /****更新人id****/ + private String updateUserId; + /****更新人姓名****/ + private String updateUser; + /****创建时间****/ + private LocalDateTime createTime; + /****更新时间****/ + private LocalDateTime updateTime; + /****删除标识0-未删除，1-已删除****/ + private Integer del; + /****名称****/ + private String name; + /****描述****/ + private String describe; + /****logo****/ + private String logo; + /****应用分类category.id****/ + private Integer categoryId; + /****应用配置仅存储应用个性化固定的配置****/ + private String params; + /****输出项****/ + private String output; + /****输出样例****/ + private String outputExample; + /****发布状态0-未发布 1-已发布****/ + private Integer flagRelease; + /****是否可作为起始节点0-不可以 1-可以****/ + private Integer flagStart; + /**** 启动url当为起始节点时，流程启动会调用此接口****/ + private String urlStart; + +} diff --git a/src/main/java/com/bfd/analyze/model/RelationsEntity.java b/src/main/java/com/bfd/analyze/model/RelationsEntity.java new file mode 100644 index 0000000..a79ab45 --- /dev/null +++ b/src/main/java/com/bfd/analyze/model/RelationsEntity.java @@ -0,0 +1,42 @@ +package com.bfd.analyze.model; + +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * 关系实体类 + * + * @author jian.mao + * @date 2023年7月5日 + * @description + */ +@Data +public class RelationsEntity { + /****主键****/ + private Integer id; + /****创建人id****/ + private String createUserId; + /****创建人姓名****/ + private String createUser; + /****更新人id****/ + private String updateUserId; + /****更新人姓名****/ + private String updateUser; + /****创建时间****/ + private LocalDateTime createTime; + /****更新时间****/ + private LocalDateTime updateTime; + /****删除标识0-未删除，1-已删除****/ + private Integer del; + /****场景idscenes.id****/ + private String scenesId; + /****起始节点app.id****/ + private String startId; + /****结束节点app.id****/ + private String endId; + /****起始节点codeapp.app_code****/ + private String startCode; + /****结束节点codeapp.app_code****/ + private String endCode; +} diff --git a/src/main/java/com/bfd/analyze/model/ScenesEntity.java b/src/main/java/com/bfd/analyze/model/ScenesEntity.java new file mode 100644 index 0000000..297c96f --- /dev/null +++ b/src/main/java/com/bfd/analyze/model/ScenesEntity.java @@ -0,0 +1,59 @@ +package com.bfd.analyze.model; + +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * 场景实体类 + * + * @author jian.mao + * @date 2023年7月5日 + * @description + */ +@Data +public class ScenesEntity { + /****主键****/ + private Integer id; + /****创建人id****/ + private String createUserId; + /****创建人姓名****/ + private String createUser; + /****更新人id****/ + private String updateUserId; + /****更新人姓名****/ + private String updateUser; + /****创建时间****/ + private LocalDateTime createTime; + /****更新时间****/ + private LocalDateTime updateTime; + /****删除标识0-未删除，1-已删除****/ + private Integer del; + /****场景名称****/ + private String scenesName; + /****场景描述****/ + private String scenesDescribe; + /****场景版本复制的新场景重置版本****/ + private Integer version; + /****场景分类category.id****/ + private Integer categoryId; + /****发布状态0-未发布 1-已发布****/ + private Integer flagRelease; + /****场景归属1-公开的 2-私有的****/ + private Integer flagOwner; + /****流程状态1-待运行 2-运行中 3-已完成****/ + private Integer status; + /****流程画布****/ + private String diagram; + /****原始场景id用于场景复制****/ + private Integer originalScenesId; + /****来源0-内部 1-采集平台****/ + private Integer sourceId; + /****外部主键****/ + private String sourceDataId; + /****单、多分支****/ + private Integer multiBranch; + /****存储数据开关****/ + private Integer flagData; + +} diff --git a/src/main/java/com/bfd/analyze/monitor/ZookeeperNodeMonitor.java b/src/main/java/com/bfd/analyze/monitor/ZookeeperNodeMonitor.java new file mode 100644 index 0000000..3f8119c --- /dev/null +++ b/src/main/java/com/bfd/analyze/monitor/ZookeeperNodeMonitor.java @@ -0,0 +1,50 @@ +package com.bfd.analyze.monitor; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.NodeCache; +import org.apache.curator.framework.recipes.cache.NodeCacheListener; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +/** + * @author jian.mao + * @date 2024年4月17日 + * @description + * + * @Component + */ +public class ZookeeperNodeMonitor { + + @Autowired + private CuratorFramework curatorFramework; + + @Value("${zookeeper.publish-node}") + private String nodePath; + + @PostConstruct + public void init() { + try { + // 创建节点监听器 + NodeCache nodeCache = new NodeCache(curatorFramework, nodePath); + nodeCache.start(); + + // 监听节点变化 + nodeCache.getListenable().addListener(new NodeCacheListener() { + @Override + public void nodeChanged() throws Exception { + byte[] data = nodeCache.getCurrentData().getData(); + String nodeData = new String(data); + System.out.println("Node data changed: " + nodeData); + // 在这里处理节点数据变化的逻辑，比如通知其他组件或者执行相应的操作 + + } + }); + } catch (Exception e) { + e.printStackTrace(); + // 异常处理 + } + } +} diff --git a/src/main/java/com/bfd/analyze/process/ErrorTaskProcess.java b/src/main/java/com/bfd/analyze/process/ErrorTaskProcess.java new file mode 100644 index 0000000..d44aa53 --- /dev/null +++ b/src/main/java/com/bfd/analyze/process/ErrorTaskProcess.java @@ -0,0 +1,75 @@ +package com.bfd.analyze.process; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.analyze.cache.ConfigCache; +import com.bfd.analyze.entity.Constants; +import com.bfd.analyze.utils.FileUtil; +import com.bfd.analyze.utils.PauseTool; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.Map; + +/** + * 错误任务处理线程 + * @author jian.mao + * @date 2023年8月15日 + * @description + */ +@Component +@Slf4j +public class ErrorTaskProcess implements Runnable { + + @Value("${task.error-task-path}") + private String path; + @Value("${task.error-time}") + private Integer maxErrorTime; + @Autowired + PauseTool pauseTool; + @Override + public void run() { + while(ConfigCache.isStart){ + Map task = null; + try { + task = ConfigCache.errorTaskQueue.take(); + Integer scense_id = (Integer) task.get(Constants.SCENES_ID); + Integer version = (Integer) task.get(Constants.VERSION); + if (!pauseTool.check(scense_id,version)) { + log.info("暂停任务：{}",JSONObject.toJSONString(task)); + continue; + } + errorTaskDealWith(task); + } catch (Exception e) { + log.error("错误任务处理失败---",e); + } + } + } + + /** + * 失败任务处理 + * @param errorTask + */ + private void errorTaskDealWith(Map errorTask){ + if(errorTask.containsKey(Constants.ERROR_TIME)){ + Integer errorTime = (Integer) errorTask.get(Constants.ERROR_TIME); + if(errorTime < maxErrorTime){ + //没到重试上限 + errorTask.put(Constants.ERROR_TIME, (errorTime+1)); + //失败任务写kafka重新流转 + + //直接放回队列会出现死锁情况 + ConfigCache.putQueue(ConfigCache.taskQueue, errorTask); + }else{ + //已达上限---写入错误文件 + FileUtil.writeFile(path, JSONObject.toJSONString(errorTask)); + } + }else{ + //第一次重试 + errorTask.put(Constants.ERROR_TIME, 1); + //失败任务写kafka重新流转 + ConfigCache.putQueue(ConfigCache.taskQueue, errorTask); + } + } +} diff --git a/src/main/java/com/bfd/analyze/process/ResultParseProcess.java b/src/main/java/com/bfd/analyze/process/ResultParseProcess.java new file mode 100644 index 0000000..109cc23 --- /dev/null +++ b/src/main/java/com/bfd/analyze/process/ResultParseProcess.java @@ -0,0 +1,365 @@ +package com.bfd.analyze.process; + +import cn.hutool.core.lang.UUID; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.analyze.cache.ConfigCache; +import com.bfd.analyze.config.BillingConfig; +import com.bfd.analyze.entity.Constants; +import com.bfd.analyze.utils.DownLoadUtil; +import com.bfd.analyze.utils.ElasticsearchUtil; +import com.bfd.analyze.utils.EncryptionUtil; +import com.bfd.analyze.utils.JsonUtil; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.HashOperations; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * 任务结果处理 + * @author jian.mao + * @date 2023年7月6日 + * @description + */ +@Component +@Slf4j +public class ResultParseProcess implements Runnable { + + @Autowired + private StringRedisTemplate stringRedisTemplate; + + @Value("${elasticsearch.index-name}") + private String indexName; + + @Value("${elasticsearch.cluster-nodes}") + private String nodes; + + @Value("${elasticsearch.username}") + private String username; + + @Value("${elasticsearch.password}") + private String password; + + @Autowired + private BillingConfig billingConfig; + @Override + public void run() { + // TODO Auto-generated method stub + while(ConfigCache.isStart){ + try { + Map result = null; + try { + result = ConfigCache.ResultQueue.take(); + } catch (InterruptedException e) { + log.error("获取调回结果失败---",e); + } + //数据新增内部唯一标识（二次推送不用新增数据id） + String dataId = UUID.randomUUID().toString(); + if(!result.containsKey(Constants.DATAID)){ + result.put(Constants.DATAID, dataId); + }else if(!result.containsKey(Constants.COVER)){ + result.put(Constants.DATAID, dataId); + } + //数据新增主体id 从流程起始到结尾都留存的id + if(!result.containsKey(Constants.DATAPROCESSID)){ + result.put(Constants.DATAPROCESSID, UUID.randomUUID().toString()); + } + //扩散类型的应用存在parentId字段 + if(result.containsKey(Constants.IS_DIFFUSION) && (boolean)result.get(Constants.IS_DIFFUSION)){ + if(!result.containsKey(Constants.PARENTID)){ + //不包含父节点，第一个扩散型应用 + List parentIds = new ArrayList(); + parentIds.add(dataId); + result.put(Constants.PARENTID, parentIds); + }else{ + //第一个之后的扩散型应用 + List parentIds = (List) result.get(Constants.PARENTID); + parentIds.add(dataId); + result.put(Constants.PARENTID, parentIds); + } + } + //新增计费 + Map appResult = (Map) result.get(Constants.REQUEST_RESULT); + if(appResult !=null && !appResult.isEmpty()){ + if(appResult.containsKey(Constants.STATUS)){ + //应用处理成功才计费 + if((int)appResult.get(Constants.STATUS) == 1){ + //指定应用进行计费 + if(billingConfig.getApps().containsKey(result.get(Constants.MODULE_ID).toString())){ + log.info("应用：{}，进行计费。",result.get(Constants.MODULE_ID)); + Billing(result); + } + } + } + } + //调试模型不考虑数据开关 + if(result.containsKey(Constants.TRACE) && (boolean)result.get(Constants.TRACE)){ + //数据持久化到es---异步写入---再考虑 + ConfigCache.putQueue(ConfigCache.saveDataQueue, result); + }else{ + //根据存储标识判断数据是否持久化 历史流程不包含这个标识FLAG_DATA，默认写入、含标识并且为1的是迭代后的版本 写入 + if(!result.containsKey(Constants.FLAG_DATA) || (Integer)result.get(Constants.FLAG_DATA) == 1){ + //数据持久化到es---异步写入---再考虑 + ConfigCache.putQueue(ConfigCache.saveDataQueue, result); + } + } + + log.info("调回信息---场景id：{}，节点id{}",result.get(Constants.SCENES_ID),result.get(Constants.APP_ID)); + List> nextAppIds = (List>) result.get(Constants.NEXT_APP_ID); + //二次推送获取完整data ---覆盖标识cover + if(result.containsKey(Constants.COVER)){ + log.info("结果修改重新进行流程处理。节点id:{}",result.get(Constants.APP_ID)); + //先获取当前数据data + Map data = (Map)result.get(Constants.DATA); + if(data == null){ + data = new HashMap(16); + } + try { + //组装data + if(!result.containsKey(Constants.DATAPROCESSID)){ + log.error("结果修改缺少必要参数：{}",Constants.DATAPROCESSID); + } + //扩散型数据修改 + if(result.containsKey(Constants.PARENTID) && result.get(Constants.PARENTID) != null){ + Map dataIds =new HashMap(16); + List parentIds = (List) result.get(Constants.PARENTID); + //没有父节点的数据源 + String requestBody = String.format("{\"query\":{\"bool\":{\"must\":[{\"match\":{\"dataProcessId\":\"%s\"}}],\"must_not\":[{\"exists\":{\"field\":\"%s\"}}],\"should\":[]}}}",(String)result.get(Constants.DATAPROCESSID),Constants.PARENTID); + Map dataProcessResult = JSONObject.parseObject(ElasticsearchUtil.getDocument(nodes,username,password,indexName,requestBody)); + Map hits = (Map) dataProcessResult.get(Constants.HITS); + List> hitList = (List>) hits.get(Constants.HITS); + if(hitList != null && hitList.size() > 0){ + for (Map map : hitList) { + Map esSource = (Map) map.get(Constants.ES_SOURCE); + if(!data.containsKey(esSource.get(Constants.APPCODE))){ + String resultStr = (String) esSource.get(Constants.REQUEST_RESULT); + data.put((String)esSource.get(Constants.APPCODE), resultStr); + //获取二次推送的节点的dataid + dataIds.put((String)esSource.get(Constants.APPCODE),esSource.get(Constants.DATAID)); + } + } + result.put(Constants.DATAIDS, dataIds); + } + log.info("~~~~~~~~非扩散型应用数据获取完毕：{}",JSONObject.toJSONString(dataIds)); + //有父节点的数据源 + dataProcessResult = JSONObject.parseObject(ElasticsearchUtil.getDocumentBySingleCondition(Constants.PARENTID,parentIds.get(0),nodes,username,password,indexName)); + hits = (Map) dataProcessResult.get(Constants.HITS); + hitList = (List>) hits.get(Constants.HITS); + if(hitList != null && hitList.size() > 0){ + for (Map map : hitList) { + Map esSource = (Map) map.get(Constants.ES_SOURCE); + if(!data.containsKey(esSource.get(Constants.APPCODE))){ + String resultStr = (String) esSource.get(Constants.REQUEST_RESULT); + data.put((String)esSource.get(Constants.APPCODE), resultStr); + //获取二次推送的节点的dataid + dataIds.put((String)esSource.get(Constants.APPCODE),esSource.get(Constants.DATAID)); + } + } + result.put(Constants.DATAIDS, dataIds); + } + log.info("`````扩散型应用数据获取完毕：{}",JSONObject.toJSONString(dataIds)); + }else{ + Map dataProcessResult = JSONObject.parseObject(ElasticsearchUtil.getDocumentBySingleCondition(Constants.DATAPROCESSID,(String)result.get(Constants.DATAPROCESSID),nodes,username,password,indexName)); + Map hits = (Map) dataProcessResult.get(Constants.HITS); + List> hitList = (List>) hits.get(Constants.HITS); + if(hitList != null && hitList.size() > 0){ + Map dataIds =new HashMap(16); + for (Map map : hitList) { + Map esSource = (Map) map.get(Constants.ES_SOURCE); + if(!data.containsKey(esSource.get(Constants.APPCODE))){ + String resultStr = (String) esSource.get(Constants.REQUEST_RESULT); + data.put((String)esSource.get(Constants.APPCODE), resultStr); + //获取二次推送的节点的dataid + dataIds.put((String)esSource.get(Constants.APPCODE),esSource.get(Constants.DATAID)); + } + } + result.put(Constants.DATAIDS, dataIds); + } + } + //data赋值 + result.put(Constants.DATA, data); + } catch (Exception e) { + log.error("获取主体数据异常：",e); + } + } + for (Map map : nextAppIds) { + //获取下一个任务节点id + String endId = (String) map.get(Constants.END_ID); + //当前节点为空或与结束节点一致则流程流转完毕9 + if(endId == null || endId .equals(result.get(Constants.APP_CODE))){ + //没有break，可能会有多个节点的扩散 + log.info("到最后一个节点了，不进行扩散---------------"); + continue; + } + //获取场景id + Integer scenesId = (Integer) result.get(Constants.SCENES_ID); + //获取下一节点任务 + log.info("nextTask redis key is:{}",scenesId+Constants.UNDERLINE+endId); + Map nextTask = JSONObject.parseObject(stringRedisTemplate.opsForValue().get(scenesId+Constants.UNDERLINE+endId)); + //结果组装 + bulidNextTask(result,nextTask); + } + } catch (Throwable e) { + log.error("未知错误---",e); + } + } + } + + /** + * 下个任务参数组装 + * @param result + * @param nextTask + */ + private void bulidNextTask(Map result,Map nextTask){ + Map newTask = new HashMap(32); + for (Entry entry:nextTask.entrySet()) { + newTask.put(entry.getKey(), entry.getValue()); + } + try { + //补充data + newTask.put(Constants.DATA, result.get(Constants.DATA)); + //二次推送标识补充， 先判断 + if(result.containsKey(Constants.COVER)){ + newTask.put(Constants.COVER, result.get(Constants.COVER)); + Map dataIds = (Map) result.get(Constants.DATAIDS); + newTask.put(Constants.DATAID, dataIds.get(newTask.get(Constants.APP_CODE))); + } + //获取当前节点请求返回结果 + Map requestResult = (Map) result.get(Constants.REQUEST_RESULT); + Map nextData = null; + if(newTask.get(Constants.DATA) == null || newTask.get(Constants.DATA).equals(Constants.EMPTY)){ + nextData = new HashMap(16); + }else{ + nextData = (Map) newTask.get(Constants.DATA); + } + //数据源类型应用 ------ 最后数据 标识组装 + if(requestResult.containsKey(Constants.ISLAST)){ + nextData.put(Constants.ISLAST, requestResult.get(Constants.ISLAST)); + } + //数据标识传递，有就传递，没有不生成-####-该调度不针对数据处理，基于项目处理，数据id根据业务需求所增 + if(result.containsKey(Constants.BUSINESSKEY)){ + newTask.put(Constants.BUSINESSKEY, result.get(Constants.BUSINESSKEY)); + nextData.put(Constants.BUSINESSKEY, result.get(Constants.BUSINESSKEY)); + } + String retStr = (String) requestResult.get(Constants.REQUEST_RESULT_RESULTS); + //data赋值此节点结果 + nextData.put((String) result.get(Constants.APP_CODE), retStr); + //多分支场景缓存到redis 结果集 + if(newTask.containsKey(Constants.MULTI_BRANCH) && (Integer)newTask.get(Constants.MULTI_BRANCH) == 1){ + String dataKey = null; + if(result.containsKey(Constants.BUSINESSKEY)){ + String businessKey = (String) result.get(Constants.BUSINESSKEY); + dataKey = businessKey+Constants.UNDERLINE+result.get(Constants.SCENES_ID)+Constants.UNDERLINE+result.get(Constants.SCENES_ID); + }else{ + dataKey = result.get(Constants.SCENES_ID)+Constants.UNDERLINE+result.get(Constants.SCENES_ID); + } + dataCache(dataKey,(String)result.get(Constants.APP_CODE),retStr); + + //all 结果汇总 + Map nextOutput = (Map) nextTask.get(Constants.OUTPUT); + if(nextOutput.containsKey(Constants.DATA)){ + Map redisData = getDataCache(dataKey); + nextData.putAll(redisData); + log.info("All api result merge"); + deleteDataCache(dataKey); + log.info("delete data cache"); + } + } + + newTask.put(Constants.DATA, nextData); + //数据流程主题id + newTask.put(Constants.DATAPROCESSID, result.get(Constants.DATAPROCESSID)); + //扩散型父节点id ----聚会类应用不需要此字段 + if(result.containsKey(Constants.PARENTID)){ + if(!result.containsKey(Constants.POLYMERIZATION) || !(boolean)result.get(Constants.POLYMERIZATION)){ + newTask.put(Constants.PARENTID, result.get(Constants.PARENTID)); + } + } + //newTask或string直接写入队列 + ConfigCache.putQueue(ConfigCache.taskQueue, newTask); + } catch (Exception e) { + // TODO: handle exception + log.error("构建次级任务失败---",e); + } + + } + /** + * data在redis缓存 + * @param dataKey + * @param key + * @param value + */ + private void dataCache(String dataKey,String key,String value){ + HashOperations hashOps = stringRedisTemplate.opsForHash(); + // Set individual fields and values + hashOps.put(dataKey, key, value); + } + + /** + * 取出redis中的data + * @param dataKey + * @return + */ + private Map getDataCache(String dataKey){ + HashOperations hashOps = stringRedisTemplate.opsForHash(); + Map map = hashOps.entries(dataKey); + return map; + } + + /** + * 删除缓存 + * @param dataKey + */ + private void deleteDataCache(String dataKey){ + stringRedisTemplate.delete(dataKey); + } + + + /** + * 计费接口调用 + * @param data + */ + private void Billing(Map data){ + Map params = new HashMap(16); + params.put(Constants.PLANCODE, data.get(Constants.PLANCODE)); + params.put(Constants.SCENESID, data.get(Constants.SCENES_ID)); + params.put(Constants.NOTE, data.get(Constants.SCENES_NAME)); + params.put(Constants.DATAID, data.get(Constants.DATAID)); + Map input = (Map) data.get(Constants.INPUT); + StringBuffer module = new StringBuffer(); + module.append(Constants.CDA_); + module.append(data.get(Constants.MODULE_ID)); + if((int)data.get(Constants.MODULE_ID) == billingConfig.getAimodelType()){ + //智能模型分多个子模型，区分处理 + module.append("_"); + module.append(input.get(Constants.MODELTYPE)); + } + params.put(Constants.MODULE, module.toString()); + log.info("接口地址：{}，接口参数：{}",billingConfig.getBillingUrl() + Constants.BILLINGSUFFIX,JSONObject.toJSONString(params)); + String response = DownLoadUtil.doPost(billingConfig.getBillingUrl() + Constants.BILLINGSUFFIX, JSONObject.toJSONString(params)); + log.info("计费接口响应结果：{}",response); + } +} diff --git a/src/main/java/com/bfd/analyze/process/ResultSendQueue.java b/src/main/java/com/bfd/analyze/process/ResultSendQueue.java new file mode 100644 index 0000000..0ae832f --- /dev/null +++ b/src/main/java/com/bfd/analyze/process/ResultSendQueue.java @@ -0,0 +1,48 @@ +package com.bfd.analyze.process; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.analyze.cache.ConfigCache; +import com.bfd.analyze.entity.Constants; +import com.bfd.analyze.utils.PauseTool; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.stereotype.Component; + +import java.util.Map; + + +/** + * 接口响应结果操作类 + * @author jian.mao + * @date 2023年7月6日 + * @description + */ +@Component +@Slf4j +public class ResultSendQueue { + @Autowired + PauseTool pauseTool; + /** + * kafka读取结果写入队列 + * @param message + */ + @KafkaListener(topics = "#{kafkaConfig.getKafkaTopic()}") + public void consumeMessage(String message) { + // 处理接收到的消息逻辑 + try { + Map result = JSONObject.parseObject(message); + Integer scense_id = (Integer) result.get(Constants.SCENES_ID); + Integer version = (Integer) result.get(Constants.VERSION); + //结果修改可不判断是否暂停 + if (result.containsKey(Constants.COVER) || pauseTool.check(scense_id,version)) { + ConfigCache.ResultQueue.put(result); + }else { + log.info("暂停任务：{}",JSONObject.toJSONString(result)); + } + } catch (Exception e) { + // TODO: handle exception + log.error("结果集json转换失败，result:{},\n",message,e); + } + } +} diff --git a/src/main/java/com/bfd/analyze/process/SaveDataProcess.java b/src/main/java/com/bfd/analyze/process/SaveDataProcess.java new file mode 100644 index 0000000..782dd66 --- /dev/null +++ b/src/main/java/com/bfd/analyze/process/SaveDataProcess.java @@ -0,0 +1,57 @@ +package com.bfd.analyze.process; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.analyze.cache.ConfigCache; +import com.bfd.analyze.entity.Constants; +import com.bfd.analyze.service.DataService; +import com.bfd.analyze.utils.PauseTool; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Map; + +/** + * 数据持久化--es + * @author jian.mao + * @date 2023年7月7日 + * @description + */ +@Component +@Slf4j +public class SaveDataProcess implements Runnable { + @Autowired + DataService dataService; + + @Autowired + PauseTool pauseTool; + @Override + public void run() { + while(ConfigCache.isStart){ + try { + Map result = null; + try { + result = ConfigCache.saveDataQueue.take(); + Integer scense_id = (Integer) result.get(Constants.SCENES_ID); + Integer version = (Integer) result.get(Constants.VERSION); + if(result.containsKey(Constants.COVER)){ + //结果修改不进行暂停任务判断 + log.info("结果修改入库---"); + }else if (!pauseTool.check(scense_id,version)) { + log.info("暂停任务：{}", JSONObject.toJSONString(result)); + continue; + } + } catch (InterruptedException e) { + log.error("获取保存数据失败---",e); + } + log.info("数据保存信息---场景id：{}，节点id{}",result.get(Constants.SCENES_ID),result.get(Constants.APP_ID)); + dataService.saveData(result); + } catch (Throwable e) { + // TODO: handle exception + log.error("未知错误---",e); + } + } + } + + +} diff --git a/src/main/java/com/bfd/analyze/process/TaskdispatchProcess.java b/src/main/java/com/bfd/analyze/process/TaskdispatchProcess.java new file mode 100644 index 0000000..1498b53 --- /dev/null +++ b/src/main/java/com/bfd/analyze/process/TaskdispatchProcess.java @@ -0,0 +1,248 @@ +package com.bfd.analyze.process; + +import cn.hutool.core.lang.UUID; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.analyze.cache.ConfigCache; +import com.bfd.analyze.entity.AnalystData; +import com.bfd.analyze.entity.Constants; +import com.bfd.analyze.utils.DateUtil; +import com.bfd.analyze.utils.DownLoadUtil; +import com.bfd.analyze.utils.ElasticsearchUtil; +import com.bfd.analyze.utils.EncryptionUtil; +import com.bfd.analyze.utils.FileUtil; +import com.bfd.analyze.utils.PauseTool; +import com.google.gson.Gson; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; + +/** + * 任务调出线程 + * @author jian.mao + * @date 2023年7月6日 + * @description + */ +@Component +@Slf4j +public class TaskdispatchProcess implements Runnable{ + @Value("${task.cycle-intv}") + private String intv; + + @Autowired + PauseTool pauseTool; + + @Value("${elasticsearch.dispatch.index-name}") + private String indexName; + + @Value("${elasticsearch.cluster-nodes}") + private String nodes; + + @Value("${elasticsearch.username}") + private String username; + + @Value("${elasticsearch.password}") + private String password; + + @Value("${task.error-result-path}") + private String path; + @Override + public void run() { + Gson gson = new Gson(); + while(ConfigCache.isStart){ + try { + Map task = null; + try { + task = ConfigCache.taskQueue.take(); + Integer scense_id = (Integer) task.get(Constants.SCENES_ID); + Integer version = (Integer) task.get(Constants.VERSION); + if(task.containsKey(Constants.COVER)){ + log.info("结果修改再次调出---"); + }else if (!pauseTool.check(scense_id,version)) { + log.info("暂停任务：{}",JSONObject.toJSONString(task)); + continue; + }else{ + //数据调出写入唯一标识 + String dataId = UUID.randomUUID().toString(); + task.put(Constants.DATAID, dataId); + //放入调出id --写入调回数据里的 + task.put(Constants.DISPATCHID, dataId); + } + } catch (InterruptedException e) { + log.error("获取调出任务失败---",e); + } +// log.info("调出任务：{}",JSONObject.toJSONString(task)); + //调用接口 ---还是需要区分module，过滤器要在调度本身 + String address = (String) task.get(Constants.ADDRESS); + //模型名称 + String module = (String) task.get(Constants.MODULE); + //调出写入es -结果修改2次推送的，不计入统计 + if(!task.containsKey(Constants.COVER)){ + //调试模型不考虑数据开关 + if(task.containsKey(Constants.TRACE) && (boolean)task.get(Constants.TRACE)){ + saveData(task); + }else{ + //根据存储标识判断数据是否持久化 历史流程不包含这个标识FLAG_DATA，默认写入、含标识并且为1的是迭代后的版本 写入 + if(!task.containsKey(Constants.FLAG_DATA) || (Integer)task.get(Constants.FLAG_DATA) == 1){ + saveData(task); + } + } + } + String content = DownLoadUtil.doPost(address, gson.toJson(task)); + log.info("调出任务，scenes_id：{}，app_id：{}，module：{}",task.get(Constants.SCENES_ID),task.get(Constants.APP_ID),module); + if(content.contains(Constants.REQUEST_ERROR_MESSAGE)){ + log.error("接口请求异常，scenes_id：{}，app_id：{}，module：{}，address：{}，errormessage：{}",task.get(Constants.SCENES_ID),task.get(Constants.APP_ID),module,address,content); + //删除调出索引的数据 + ElasticsearchUtil.deleteDataById((String)task.get(Constants.DATAID), nodes, username, password, indexName); + //错误任务-处理 + ConfigCache.errorTaskQueue.push(task); + }else{ + log.info("接口返回值：{}",content); + + } + //针对周期性app处理 +// if((Integer)task.get(Constants.SCHEDULING_TYPE)==1){ +// task.put(Constants.NEXT_SCHEDULING_TIME,System.currentTimeMillis()+intv); +// //周期任务 +// ConfigCache.cycleTaskWaitQueue.put(task); +// } + } catch (Throwable e) { + log.error("未知错误----",e); + } + } + } + + + + public void saveData(Map data) { + // TODO Auto-generated method stub + try { + AnalystData analystData = new AnalystData(); + analystData.setAppId((Integer) data.get(Constants.APP_ID)); + analystData.setAppCode((String) data.get(Constants.APP_CODE)); + analystData.setCreated(DateUtil.getDateTime()); + analystData.setLastEdit(DateUtil.getDateTime()); + analystData.setTransferId((Integer) data.get(Constants.TRANSFER_ID)); + analystData.setAppName((String) data.get(Constants.APP_NAME)); + analystData.setAppDescribe((String) data.get(Constants.DESCRIBE)); + analystData.setModule((String) data.get(Constants.MODULE)); + analystData.setNextAppId((List>) data.get(Constants.NEXT_APP_ID)); + analystData.setStartTag((Boolean) data.get(Constants.START_TAG)); + analystData.setScenesId((Integer) data.get(Constants.SCENES_ID)); + analystData.setBusinessKey(data.containsKey(Constants.BUSINESSKEY)?data.get(Constants.BUSINESSKEY).toString():Constants.EMPTY); + if (data.containsKey(Constants.POLYMERIZATION) && (boolean)data.get(Constants.POLYMERIZATION)){ + //多合一类应用调出数据只保留一条 + String dataId = data.get(Constants.APP_CODE).toString(); + analystData.setDataId(dataId); + data.put(Constants.DATAID, dataId); + //放入调出id --写入调回数据里的 + data.put(Constants.DISPATCHID, dataId); + log.info("多合一类应用调出：{}",dataId); + }else{ + analystData.setDataId(data.containsKey(Constants.DATAID)?data.get(Constants.DATAID).toString():UUID.randomUUID().toString()); + } + analystData.setDel(0); + if(data.containsKey(Constants.DEFAULTCOMPLETION) && (boolean)data.get(Constants.DEFAULTCOMPLETION)){ + //有些应用不需要调回去同步完成状态，比如过滤器，直接完成 + analystData.setStatus(1); + }else{ + analystData.setStatus(0); + } + //trace，0正式数据，1是测试数据 + if(data.containsKey(Constants.TRACE) && (boolean)data.get(Constants.TRACE)){ + analystData.setTrace(1); + }else{ + analystData.setTrace(0); + } + //es数据添加version 用于删除判断 + Integer version = (Integer) data.get(Constants.VERSION); + analystData.setVersion(version); + boolean status = save(analystData); + if(!status){ + FileUtil.writeFile(path, JSONObject.toJSONString(data)); + } + } catch (Exception e) { + log.error("调出任务转换es实体失败----",e); + } +// } + + } + + + /** + * 节点数据持久化 + * @param analystData + * @return + */ + private boolean save(AnalystData analystData){ + //只写入一个节点 + String node = nodes.split(",")[0]; + // 创建凭据提供者 + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + // 创建一个 HttpClient，设置凭据提供者 + CloseableHttpClient httpClient = null; + try { + // 创建一个 HTTP POST 请求用于写入数据到索引 + if(username != null && !username.trim().equals(Constants.EMPTY)){ + // 创建凭据提供者 + httpClient = HttpClients.custom() + .setDefaultCredentialsProvider(credentialsProvider) + .build(); + }else{ + httpClient = HttpClients.custom().build(); + } + + StringBuffer host = new StringBuffer("http://"); + host.append(node); + host.append("/"); + host.append(indexName); + host.append("/_doc/"); + // Elasticsearch主机、索引名称和文档ID + host.append(analystData.getDataId()); + HttpPost httpPost = new HttpPost(host.toString()); + // 设置请求体，包含要写入的文档数据 + StringEntity entity = new StringEntity(JSONObject.toJSONString(analystData), ContentType.APPLICATION_JSON); + httpPost.setEntity(entity); + // 发送请求并获取响应 + HttpResponse response = httpClient.execute(httpPost); + // 处理响应 + int statusCode = response.getStatusLine().getStatusCode(); + String responseBody = EntityUtils.toString(response.getEntity()); + int code = 201; + int updateCode = 200; + if (statusCode == code) { + log.info("调出任务数据成功写入到索引：{}，文档ID：{}，appid：{}",indexName,analystData.getDataId(),analystData.getAppId()); + return true; + }else if(statusCode == updateCode){ + log.info("调出任务数据成功更新到索引：{}，文档ID：{}，appid：{}",indexName,analystData.getDataId(),analystData.getAppId()); + return true; + } else { + log.error("调出任务数据写入失败：{}，文档ID：{}，appid：{}，es响应内容：{}",indexName,analystData.getDataId(),analystData.getAppId(),responseBody); + return false; + } + } catch (Exception e) { + log.error("调出任务数据写入异常：",e); + return false; + } + + + } +} diff --git a/src/main/java/com/bfd/analyze/service/DataService.java b/src/main/java/com/bfd/analyze/service/DataService.java new file mode 100644 index 0000000..f39bb50 --- /dev/null +++ b/src/main/java/com/bfd/analyze/service/DataService.java @@ -0,0 +1,18 @@ +package com.bfd.analyze.service; + +import java.util.Map; + + +/** + * 数据处理业务层 + * @author jian.mao + * @date 2023年7月11日 + * @description + */ +public interface DataService { + + /** + * 数据存储 + */ + public void saveData(Map data); +} diff --git a/src/main/java/com/bfd/analyze/service/DistributedLockService.java b/src/main/java/com/bfd/analyze/service/DistributedLockService.java new file mode 100644 index 0000000..4a0085b --- /dev/null +++ b/src/main/java/com/bfd/analyze/service/DistributedLockService.java @@ -0,0 +1,23 @@ +package com.bfd.analyze.service; + +/** + * + * @author jian.mao + * @date 2023年7月25日 + * @description + */ + +public interface DistributedLockService { + + /** + * redis加锁 + * @return + */ + public boolean acquireLock(); + + /** + * 释放锁 + * redis + */ + public void releaseLock(); +} diff --git a/src/main/java/com/bfd/analyze/service/ProcessTriggerService.java b/src/main/java/com/bfd/analyze/service/ProcessTriggerService.java new file mode 100644 index 0000000..7337b64 --- /dev/null +++ b/src/main/java/com/bfd/analyze/service/ProcessTriggerService.java @@ -0,0 +1,59 @@ +package com.bfd.analyze.service; + +/** + * @PROJECT_NAME: analyst_assistant_schedule + * @DESCRIPTION: + * @AUTHOR: jian.mao + * @DATE: 2023/10/20 17:28 + */ +public interface ProcessTriggerService { + + /** + * @param param + * @return + */ + public String start(String param); + + + /** + * @param param + * @return + */ + public String stop(String param); + + + /** + * @param dataJson + * @return + */ + public String active(String param); + + + /** + * @param dataJson + * @return + */ + public String pause(String dataJson); + + + /** + * @param dataJson + * @return + */ + public String delete(String dataJson); + + + /** + * @param dataJson + * @return + */ + public String retry(String dataJson); + + + /** + * @param dataJson + * @return + */ + public String scenesDelete(String dataJson); + +} diff --git a/src/main/java/com/bfd/analyze/service/impl/DataServiceImpl.java b/src/main/java/com/bfd/analyze/service/impl/DataServiceImpl.java new file mode 100644 index 0000000..8079fc3 --- /dev/null +++ b/src/main/java/com/bfd/analyze/service/impl/DataServiceImpl.java @@ -0,0 +1,240 @@ +package com.bfd.analyze.service.impl; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.analyze.config.BillingConfig; +import com.bfd.analyze.entity.AnalystData; +import com.bfd.analyze.entity.Constants; +import com.bfd.analyze.service.DataService; +import com.bfd.analyze.utils.DateUtil; +import com.bfd.analyze.utils.DownLoadUtil; +import com.bfd.analyze.utils.ElasticsearchUtil; +import com.bfd.analyze.utils.FileUtil; + +/** + * 数据处理业务层接口实现类 + * @author jian.mao + * @date 2023年7月11日 + * @description + */ +@Service +@Slf4j +public class DataServiceImpl implements DataService { + + @Value("${elasticsearch.index-name}") + private String indexName; + + @Value("${elasticsearch.dispatch.index-name}") + private String dispatchIndexName; + + @Value("${elasticsearch.cluster-nodes}") + private String nodes; + + @Value("${elasticsearch.username}") + private String username; + + @Value("${elasticsearch.password}") + private String password; + + @Value("${task.error-result-path}") + private String path; + + @SuppressWarnings("unchecked") + @Override + public void saveData(Map data) { + // TODO Auto-generated method stub + Map result = (Map) data.get(Constants.REQUEST_RESULT); +// log.info("要保存的数据----{}",JSONObject.toJSONString(data)); + if(result == null || result.isEmpty()){ + log.error("返回数据无响应结果-----场景id：{}，节点id{}",data.get(Constants.SCENES_ID),data.get(Constants.APP_ID)); + return ; + } + //获取当前节点请求返回结果 + String retStr = (String) result.get(Constants.REQUEST_RESULT_RESULTS); + if(retStr == null || retStr.equals(Constants.EMPTY)){ + log.error("响应结果无数据体-----场景id：{}，节点id{}",data.get(Constants.SCENES_ID),data.get(Constants.APP_ID)); +// return ; + } + //过滤器内容干掉 +// if(data.get(Constants.BLUEPRINT_NAME).equals(Constants.FILTER_ZH)){ +// return; +// } +// String dataType = JsonUtil.checkJsonType(retStr); + //如果是list，遍历分多条任务发送 +// if(dataType.equals(Constants.LIST_TYPE)){ +// List results = JSONObject.parseArray(retStr); +// for (Object object : results) { +// try { +// AnalystData analystData = new AnalystData(); +// analystData.setAppId((Integer) data.get(Constants.APP_ID)); +// analystData.setAppCode((String) data.get(Constants.APP_CODE)); +// analystData.setCreated(DateUtil.getDateTime()); +// analystData.setLastEdit(DateUtil.getDateTime()); +// analystData.setTransferId((Integer) data.get(Constants.TRANSFER_ID)); +// analystData.setAppName((String) data.get(Constants.APP_NAME)); +// analystData.setAppDescribe((String) data.get(Constants.DESCRIBE)); +// analystData.setModule((String) data.get(Constants.MODULE)); +// analystData.setResult(JSONObject.toJSONString(object)); +// analystData.setNextAppId((List>) data.get(Constants.NEXT_APP_ID)); +// analystData.setStartTag((Boolean) data.get(Constants.START_TAG)); +// analystData.setScenesId((Integer) data.get(Constants.SCENES_ID)); +// analystData.setBusinessKey(data.containsKey(Constants.BUSINESSKEY)?data.get(Constants.BUSINESSKEY).toString():Constants.EMPTY); +// analystData.setDataId((String)data.get(Constants.DATAID)); +// analystData.setDataProcessId((String)data.get(Constants.DATAPROCESSID)); +// boolean status = save(analystData); +// if(!status){ +// FileUtil.writeFile(path, JSONObject.toJSONString(data)); +// } +// } catch (Exception e) { +// log.error("数据响应体转换es实体失败----",e); +// } +// } +// }else{ + //map或string直接写入队列 + try { + AnalystData analystData = new AnalystData(); + analystData.setAppId((Integer) data.get(Constants.APP_ID)); + analystData.setAppCode((String) data.get(Constants.APP_CODE)); + analystData.setCreated(String.valueOf(System.currentTimeMillis())); + analystData.setLastEdit(String.valueOf(System.currentTimeMillis())); + analystData.setTransferId((Integer) data.get(Constants.TRANSFER_ID)); + analystData.setAppName((String) data.get(Constants.APP_NAME)); + analystData.setAppDescribe((String) data.get(Constants.DESCRIBE)); + analystData.setModule((String) data.get(Constants.MODULE)); + analystData.setResult(retStr==null?Constants.EMPTY:retStr); + analystData.setNextAppId((List>) data.get(Constants.NEXT_APP_ID)); + analystData.setStartTag((Boolean) data.get(Constants.START_TAG)); + analystData.setScenesId((Integer) data.get(Constants.SCENES_ID)); + analystData.setBusinessKey(data.containsKey(Constants.BUSINESSKEY)?data.get(Constants.BUSINESSKEY).toString():Constants.EMPTY); + analystData.setDataId((String)data.get(Constants.DATAID)); + analystData.setDataProcessId((String)data.get(Constants.DATAPROCESSID)); + analystData.setDispatchId(data.containsKey(Constants.DISPATCHID)?data.get(Constants.DISPATCHID).toString():Constants.EMPTY); + analystData.setDel(0); + if(result.containsKey(Constants.SEARCHFIELDS)){ + analystData.setVariable((List>) result.get(Constants.SEARCHFIELDS)); + } + if(result.containsKey(Constants.STATUS)){ + analystData.setStatus((int)result.get(Constants.STATUS)); + analystData.setMessage((String) result.get(Constants.MESSAGE)); + } + //有parentId就写入 + if(data.containsKey(Constants.PARENTID)){ + if(!data.containsKey(Constants.POLYMERIZATION) || !(boolean)data.get(Constants.POLYMERIZATION)){ + analystData.setParentId((List)data.get(Constants.PARENTID)); + } + } + //trace，0正式数据，1是测试数据 + if(data.containsKey(Constants.TRACE) && (boolean)data.get(Constants.TRACE)){ + analystData.setTrace(1); + }else{ + analystData.setTrace(0); + } + //es数据添加version 用于删除判断 + Integer version = (Integer) data.get(Constants.VERSION); + analystData.setVersion(version); + boolean status = save(analystData); + if(!status){ + FileUtil.writeFile(path, JSONObject.toJSONString(data)); + } + //更新调出完成状态 + try { + JSONObject results = JSONObject.parseObject(retStr); + if(results.containsKey(Constants.ISLAST)){ + //数据已调回，通知调出索引 + if(data.containsKey(Constants.DISPATCHID)){ + String updateResultStr = ElasticsearchUtil.updateFieldById(dispatchIndexName, data.get(Constants.DISPATCHID).toString(),username, password, Constants.STATUS, 1, nodes); + log.info("数据已调回，更新调出完成状态：{}",updateResultStr); + } + } + } catch (Exception e) { + // TODO: handle exception + log.error("更新调出完成状态失败：{}",JSONObject.toJSONString(data)); + } + } catch (Exception e) { + log.error("数据响应体转换es实体失败----",e); + } +// } + + } + + + /** + * 节点数据持久化 + * @param analystData + * @return + */ + private boolean save(AnalystData analystData){ + //只写入一个节点 + String node = nodes.split(",")[0]; + // 创建凭据提供者 + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + // 创建一个 HttpClient，设置凭据提供者 + CloseableHttpClient httpClient = null; + try { + // 创建一个 HTTP POST 请求用于写入数据到索引 + if(username != null && !username.trim().equals(Constants.EMPTY)){ + // 创建凭据提供者 + httpClient = HttpClients.custom() + .setDefaultCredentialsProvider(credentialsProvider) + .build(); + }else{ + httpClient = HttpClients.custom().build(); + } + + StringBuffer host = new StringBuffer("http://"); + host.append(node); + host.append("/"); + host.append(indexName); + host.append("/_doc/"); + // Elasticsearch主机、索引名称和文档ID + host.append(analystData.getDataId()); + HttpPost httpPost = new HttpPost(host.toString()); + // 设置请求体，包含要写入的文档数据 + StringEntity entity = new StringEntity(JSONObject.toJSONString(analystData), ContentType.APPLICATION_JSON); + httpPost.setEntity(entity); + // 发送请求并获取响应 + HttpResponse response = httpClient.execute(httpPost); + // 处理响应 + int statusCode = response.getStatusLine().getStatusCode(); + String responseBody = EntityUtils.toString(response.getEntity()); + int code = 201; + int updateCode = 200; + if (statusCode == code) { + log.info("数据成功写入到索引：{}，文档ID：{}，appid：{}",indexName,analystData.getDataId(),analystData.getAppId()); + return true; + }else if(statusCode == updateCode){ + log.info("数据成功更新到索引：{}，文档ID：{}，appid：{}",indexName,analystData.getDataId(),analystData.getAppId()); + return true; + } else { + log.error("数据写入失败：{}，文档ID：{}，appid：{}，es响应内容：{}",indexName,analystData.getDataId(),analystData.getAppId(),responseBody); + return false; + } + } catch (Exception e) { + log.error("数据写入异常：",e); + return false; + } + + + } +} diff --git a/src/main/java/com/bfd/analyze/service/impl/DistributedLockServiceImpl.java b/src/main/java/com/bfd/analyze/service/impl/DistributedLockServiceImpl.java new file mode 100644 index 0000000..02452e7 --- /dev/null +++ b/src/main/java/com/bfd/analyze/service/impl/DistributedLockServiceImpl.java @@ -0,0 +1,40 @@ +package com.bfd.analyze.service.impl; + +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; + +import com.bfd.analyze.entity.Constants; +import com.bfd.analyze.service.DistributedLockService; + +/** + * @author jian.mao + * @date 2023年7月31日 + * @description + */ +@Service +@Slf4j +public class DistributedLockServiceImpl implements DistributedLockService { + + @Autowired + private StringRedisTemplate redisTemplate; + + + @Override + public boolean acquireLock() { + Boolean isLocked = redisTemplate.opsForValue().setIfAbsent(Constants.LOCK_KEY, + "locked", Constants.LOCK_EXPIRE_TIME, TimeUnit.MILLISECONDS); + return isLocked != null && isLocked; + } + + @Override + public void releaseLock() { + // TODO Auto-generated method stub + redisTemplate.delete(Constants.LOCK_KEY); + } + +} diff --git a/src/main/java/com/bfd/analyze/service/impl/ProcessTriggerServiceImpl.java b/src/main/java/com/bfd/analyze/service/impl/ProcessTriggerServiceImpl.java new file mode 100644 index 0000000..7684304 --- /dev/null +++ b/src/main/java/com/bfd/analyze/service/impl/ProcessTriggerServiceImpl.java @@ -0,0 +1,637 @@ +package com.bfd.analyze.service.impl; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.analyze.cache.ConfigCache; +import com.bfd.analyze.entity.Constants; +import com.bfd.analyze.mapper.AppsMapper; +import com.bfd.analyze.mapper.ModulesMapper; +import com.bfd.analyze.mapper.RelationsMapper; +import com.bfd.analyze.mapper.ScenesMapper; +import com.bfd.analyze.model.AppsEntity; +import com.bfd.analyze.model.ModulesEntity; +import com.bfd.analyze.model.RelationsEntity; +import com.bfd.analyze.model.ScenesEntity; +import com.bfd.analyze.service.ProcessTriggerService; +import com.bfd.analyze.utils.ElasticsearchUtil; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +import java.io.IOException; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +/** + * @PROJECT_NAME: analyst_assistant_schedule + * @DESCRIPTION: + * @AUTHOR: jian.mao + * @DATE: 2023/10/20 17:37 + */ +@Slf4j +@Service +public class ProcessTriggerServiceImpl implements ProcessTriggerService { + + @Resource + private AppsMapper appsMapper; + @Resource + private ModulesMapper modulesMapper; + @Resource + private RelationsMapper relationsMapper; + @Resource + private ScenesMapper scenesMapper; + @Autowired + private StringRedisTemplate stringRedisTemplate; + @Autowired + private CuratorFramework curatorFramework; + @Value("${zookeeper.publish-node}") + private String publishNode; + private InterProcessMutex lock; + + @Value("${elasticsearch.index-name}") + private String indexName; + + @Value("${elasticsearch.dispatch.index-name}") + private String dispatchIndexName; + + @Value("${elasticsearch.cluster-nodes}") + private String nodes; + + @Value("${elasticsearch.username}") + private String username; + + @Value("${elasticsearch.password}") + private String password; + + @PostConstruct + public void init() { + lock = new InterProcessMutex(curatorFramework, publishNode); + } + + @Override + public String start(String param) { + Map response = new HashMap(16); + Map zkResult = new HashMap(16); + int code = 200; + String message = "success"; + int version = 0; + try { + Map params = JSONObject.parseObject(param); + Integer scenesId = Integer.valueOf(params.get(Constants.SCENES_ID).toString()); + //先清除缓存 + log.info("清除场景：{} 的流程缓存", scenesId); + deleteKeysWithPrefix(scenesId + Constants.UNDERLINE); + String scenes_version = stringRedisTemplate.opsForValue().get(Constants.VERSION + Constants.UNDERLINE + scenesId); + log.info("清除场景：{} ,verison不等于 {} 的历史数据", scenesId,scenes_version); + //异步删除 + Thread thread = new Thread(() -> { + log.info("流程开启--->异步删除历史数据----start"); + //调回数据删除 + ElasticsearchUtil.deleteDataBycondition(Constants.SCENESID, scenesId, nodes, username, password, indexName,scenes_version); + //调出数据删除 + ElasticsearchUtil.deleteDataBycondition(Constants.SCENESID, scenesId, nodes, username, password, dispatchIndexName,scenes_version); + log.info("流程开启--->异步删除历史数据----end"); + }); + // 启动线程 + thread.start(); + + // 尝试获取分布式锁，启动通知应用 + if (lock.acquire(Constants.LOCKTIME, TimeUnit.SECONDS)) { + try { + log.info("获取锁成功------------{}", param); + //zk 通知 启动的场景 和 版本 + zkResult.put("scenes_id", scenesId); + zkResult.put("operation", Constants.START); + //从redis尝试获取version,根据是否有version,判断是否是重新启动的流程 + if (scenes_version != null) { + //重新启动的流程 拿到version + version = Integer.parseInt(scenes_version); + } else { + //场景第一次启动 将version存储在redis + stringRedisTemplate.opsForValue().set(Constants.VERSION + Constants.UNDERLINE + scenesId, String.valueOf(version)); + //将 场景版本号 存储到内存中，便于去除暂停任务 + } + zkResult.put("version", version); + String value = JSONObject.toJSONString(zkResult); + log.info("zk通知消息-----------{}",value); +// String newKey = scenesId + Constants.UNDERLINE + version; +// // 移除CACHE中所有以scenesId开头的key +// PauseTool.CACHE.keySet().removeIf(key -> key.startsWith(scenesId + Constants.UNDERLINE)); +// PauseTool.CACHE.put(newKey, version); + //通知应用 将场景_版本号 写到缓存里 + if (curatorFramework.checkExists().forPath(publishNode) == null) { + curatorFramework.create().creatingParentsIfNeeded().forPath(publishNode, value.getBytes()); + } else { + // 如果节点存在，则更新节点数据 + curatorFramework.setData().forPath(publishNode, value.getBytes()); + } + message = "Node updated successfully"; + } finally { + // 释放锁 + lock.release(); + } + //获取场景信息 + ScenesEntity scenes = scenesMapper.getscenesById(scenesId); + //获取场景下的节点信息 + List apps = appsMapper.getAppByscenesId(scenesId); + if (apps != null && apps.size() > 0) { + Map firstTask = new HashMap(16); + for (AppsEntity app : apps) { + //获取modules + ModulesEntity module = modulesMapper.getModuleByModuleId(app.getModuleId()); + //获取relations + List relations = relationsMapper.getRelationsByscenesId(app.getScenesId()); + log.info("组装节点：{}，组装参数：{}，", JSONObject.toJSONString(app), JSONObject.toJSONString(params)); + //构建任务 + Map task = bulidTask(app, module, relations, params, scenes, version); + //任务发送 + send(task, firstTask); + } + //首节点直接调出 + if (!firstTask.isEmpty()) { + try { + //调试场景优先调出 + if(params.containsKey(Constants.TRACE) && (boolean)params.get(Constants.TRACE)){ + log.info("调式任务写入队列-----------"); + ConfigCache.taskQueue.putFirst(firstTask); + }else{ + ConfigCache.taskQueue.put(firstTask); + } + } catch (InterruptedException e) { + log.error("任务写入队列异常------"); + } + } + } else { + //查询不到流程 + code = 100001; + message = "查询不到流程"; + } + } else { + code = 100001; + message = "Failed to acquire lock"; + log.warn("获取锁失败-----------{}", param); + } + + } catch (Exception e) { + code = 100002; + message = "流程节点组装异常"; + log.error("异常信息，", e); + } + response.put(Constants.CODE, code); + response.put(Constants.MESSAGE, message); + response.put(Constants.VERSION, version); + return JSONObject.toJSONString(response); + } + + private void send(Map task, Map firstTask) { + if ((Integer) task.get(Constants.TRANSFER_ID) > 0) { + log.info("次级任务写入缓存中等待：场景--{}，app节点--{}", task.get(Constants.SCENES_ID), task.get(Constants.APP_NAME)); + log.info("次级节点任务：{}", JSONObject.toJSONString(task)); + //redis缓存###key存储规则：蓝图scenesId_appId + stringRedisTemplate.opsForValue().set(task.get(Constants.SCENES_ID) + Constants.UNDERLINE + task.get(Constants.APP_CODE), JSONObject.toJSONString(task)); + } else { + //判断是否主动调用的应用，配置流程后设置数据的应用 ，比如文件上传 + Map input = (Map) task.get(Constants.INPUT); + if (input.containsKey(Constants.IS_ACTIVE) && (boolean) input.get(Constants.IS_ACTIVE)) { + log.info("该节点为主动触发应用：场景--{}，app节点--{}", task.get(Constants.SCENES_ID), task.get(Constants.APP_NAME)); + } else { + for (Entry entry : task.entrySet()) { + firstTask.put(entry.getKey(), entry.getValue()); + } + } + } + } + + /** + * 删除缓存key + * + * @param prefix + */ + public void deleteKeysWithPrefix(String prefix) { + Set keys = stringRedisTemplate.keys(prefix + Constants.WILDCARD); + if (keys != null) { + keys.forEach(key -> stringRedisTemplate.delete(key)); + } + } + + private Map bulidTask(AppsEntity app, ModulesEntity module, List relations, Map webParams, ScenesEntity scenes, int version) { + Map task = new HashMap(16); + //调试模型所需字段 + if(webParams.containsKey(Constants.TRACE) && (boolean)webParams.get(Constants.TRACE)){ + log.info("调试字段写入：{}",JSONObject.toJSONString(webParams)); + task.put(Constants.TRACE, webParams.get(Constants.TRACE)); + task.put(Constants.DATAPROCESSID, webParams.get(Constants.DATAPROCESSID)); + } + //增加外部主键 + task.put(Constants.SOURCE_DATA_ID, scenes.getSourceDataId()); + //场景名称 + task.put(Constants.SCENES_NAME, scenes.getScenesName()); + //场景数据存储开关 + task.put(Constants.FLAG_DATA, scenes.getFlagData()); + //套餐标识 + task.put(Constants.PLANCODE, webParams.get(Constants.PLANCODE)); + //应用id + task.put(Constants.MODULE_ID, module.getId()); + //增加操作用户id + task.put(Constants.CREATEUSERID, app.getCreateUserId()); + //输出配置 ---需要解析 + Map afterOutput = new HashMap(16); + String parseBeforOpStr = app.getOutput(); + Map beforOutput = JSONObject.parseObject(parseBeforOpStr); + if (beforOutput.containsKey(Constants.FORM)) { + List> form = (List>) beforOutput.get(Constants.FORM); + for (Map map : form) { + String field = (String) map.get(Constants.FIELD); + afterOutput.put(field, field); + } + } + task.put(Constants.OUTPUT, afterOutput); + /***输入配置 = 固定配置+选配置***/ + //固定配置 + Map fixedParams = JSONObject.parseObject(module.getParams()); + //选配置 + Map input = JSONObject.parseObject(app.getInput()); + //合并 + fixedParams.putAll(input); +// input.putAll(fixedParams); + task.put(Constants.INPUT, fixedParams); + //接口地址 + task.put(Constants.ADDRESS, fixedParams.get(Constants.ADDRESS)); + //多合一应用标识 + task.put(Constants.POLYMERIZATION, fixedParams.containsKey(Constants.POLYMERIZATION)?true:false); + //不设置完成状态标识 + task.put(Constants.DEFAULTCOMPLETION, fixedParams.containsKey(Constants.DEFAULTCOMPLETION)?true:false); + //应用扩散标识 + task.put(Constants.IS_DIFFUSION, fixedParams.containsKey(Constants.IS_DIFFUSION)?true:false); + //模块名称 + task.put(Constants.MODULE, module.getName()); + //流转内部节点id + task.put(Constants.TRANSFER_ID, app.getTransferId()); + //场景id + task.put(Constants.SCENES_ID, app.getScenesId()); + //节点名称 + task.put(Constants.APP_NAME, app.getName()); + //businessKey +// task.put(Constants.BUSINESSKEY,webParams.get(Constants.BUSINESSKEY)); + task.put(Constants.BUSINESSKEY, UUID.randomUUID().toString()); + //app调出模式 + task.put(Constants.SCHEDULING_TYPE, app.getSchedulingType()); + //ida + task.put(Constants.APP_ID, app.getId()); + //节点描述 + task.put(Constants.DESCRIBE, app.getDescribe()); + //appCode + task.put(Constants.APP_CODE, app.getAppCode()); + //开始节点start_tag + task.put(Constants.START_TAG, app.getTransferId() == 0 ? true : false); + //等待节点 + task.put(Constants.WAIT_CONDITION, app.getWaitCondition()); + //next_app + task.put(Constants.RELATIONS, relations); + List> nextAppIds = new ArrayList>(); + for (RelationsEntity relation : relations) { + if (relation.getStartCode().equals(app.getAppCode())) { + Map nextApp = new HashMap(16); + nextApp.put(Constants.EDGE_ID, relation.getId()); + nextApp.put(Constants.START_ID, relation.getStartCode()); + nextApp.put(Constants.END_ID, relation.getEndCode()); + nextAppIds.add(nextApp); + } + } + task.put(Constants.NEXT_APP_ID, nextAppIds); + task.put(Constants.VERSION, version); + + return task; + } + + @Override + public String stop(String param) { + Map response = new HashMap(16); + int code = 200; + String message = "success"; + try { + // 尝试获取分布式锁 + if (lock.acquire(Constants.LOCKTIME, TimeUnit.SECONDS)) { + try { + JSONObject data = JSONObject.parseObject(param); + String value = data.get(Constants.SCENES_ID).toString(); + log.info("获取锁成功------------{}", param); + if (curatorFramework.checkExists().forPath(publishNode) == null) { + curatorFramework.create().creatingParentsIfNeeded().forPath(publishNode, value.getBytes()); + } else { + // 如果节点存在，则更新节点数据 + curatorFramework.setData().forPath(publishNode, value.getBytes()); + } + message = "Node updated successfully"; + } finally { + // 释放锁 + lock.release(); + } + } else { + code = 100001; + message = "Failed to acquire lock"; + log.warn("获取锁失败-----------{}", param); + } + } catch (Exception e) { + log.error("未知异常：{}，{}", param, e); + code = 100002; + message = "Failed to update node: " + e.getMessage(); + } + response.put(Constants.CODE, code); + response.put(Constants.MESSAGE, message); + return JSONObject.toJSONString(response); + } + + @Override + public String active(String param) { + + Map response = new HashMap(16); + int code = 200; + String message = "success"; + try { + Map params = JSONObject.parseObject(param); + Integer scenesId = Integer.valueOf(params.get(Constants.SCENES_ID).toString()); + String appCode = params.get(Constants.APP_CODE).toString(); + //获取场景信息 + ScenesEntity scenes = scenesMapper.getscenesById(scenesId); + //获取场景下的节点信息 + AppsEntity app = appsMapper.getAppByAppCode(appCode); + if (app != null) { + //获取modules + ModulesEntity module = modulesMapper.getModuleByModuleId(app.getModuleId()); + //获取relations + List relations = relationsMapper.getRelationsByscenesId(app.getScenesId()); + //设置输入配置 + Map appInput = JSONObject.parseObject(app.getInput()); + if(params.containsKey(Constants.INPUT)){ + Map input = (Map) params.get(Constants.INPUT); + appInput.putAll(input); + } + app.setInput(JSONObject.toJSONString(appInput)); + //构建任务 + + int version = 0; + String scenes_version = stringRedisTemplate.opsForValue().get(Constants.VERSION + Constants.UNDERLINE + scenesId); + if (scenes_version != null) { + //重新启动的流程 拿到version + version = Integer.parseInt(scenes_version); + } + Map task = bulidTask(app, module, relations, params, scenes, version); + //判断是否有数据体 ---系统参数 + if (params.containsKey(Constants.DATA)) { + task.put(Constants.DATA, params.get(Constants.DATA)); + } + //api接口结果传输 + if(params.containsKey(Constants.REQUEST_RESULT)){ + Object result = params.get(Constants.REQUEST_RESULT); + task.put(Constants.REQUEST_RESULT, result); + } + //任务发送 + log.info("流程主动触发：{}，组装参数：{}，", JSONObject.toJSONString(app), JSONObject.toJSONString(params)); + try { + ConfigCache.taskQueue.put(task); + } catch (InterruptedException e) { + log.error("任务写入队列异常------"); + } + } else { + //查询不到流程 + code = 100001; + message = "查询不到流程"; + } + } catch (Exception e) { + code = 100002; + message = "流程节点组装异常"; + log.error("异常信息，", e); + } + response.put(Constants.CODE, code); + response.put(Constants.MESSAGE, message); + return JSONObject.toJSONString(response); + } + + @Override + public String pause(String param) { + Map response = new HashMap(16); + Map zkResult = new HashMap(16); + int code = 200; + String message = "success"; + try { + // 尝试获取分布式锁 + if (lock.acquire(Constants.LOCKTIME, TimeUnit.SECONDS)) { + try { + JSONObject data = JSONObject.parseObject(param); + //场景id + Integer scenes_id = Integer.valueOf(data.get(Constants.SCENES_ID).toString()); + //zk 通知 停止的场景 和 版本 + zkResult.put("scenes_id", scenes_id); + zkResult.put("operation", Constants.STOP); + String scenes_version = stringRedisTemplate.opsForValue().get(Constants.VERSION + Constants.UNDERLINE + scenes_id); + int version = scenes_version != null ? Integer.parseInt(scenes_version) + 1 : 0; + stringRedisTemplate.opsForValue().set(Constants.VERSION + Constants.UNDERLINE + scenes_id, String.valueOf(version)); + zkResult.put("version", version); +// String newKey = scenes_id + Constants.SCENES_ID + version; + // 移除CACHE中所有以scenesId开头的key +// PauseTool.CACHE.keySet().removeIf(key -> key.startsWith(scenes_id + Constants.UNDERLINE)); + // 将新的key放入CACHE +// PauseTool.CACHE.put(newKey, String.valueOf(version)); + + String value = JSONObject.toJSONString(zkResult); + log.info("获取锁成功------------{}", param); + log.info("zk通知消息-----------{}",value); + if (curatorFramework.checkExists().forPath(publishNode) == null) { + curatorFramework.create().creatingParentsIfNeeded().forPath(publishNode, value.getBytes()); + } else { + // 如果节点存在，则更新节点数据 + curatorFramework.setData().forPath(publishNode, value.getBytes()); + } + message = "Node updated successfully"; + } finally { + // 释放锁 + lock.release(); + } + } else { + code = 100001; + message = "Failed to acquire lock"; + log.warn("获取锁失败-----------{}", param); + } + } catch (Exception e) { + log.error("未知异常：{}，{}", param, e); + code = 100002; + message = "Failed to update node: " + e.getMessage(); + } + response.put(Constants.CODE, code); + response.put(Constants.MESSAGE, message); + return JSONObject.toJSONString(response); + } + + @Override + public String delete(String dataJson) { + // TODO Auto-generated method stub + JSONObject parseObject = JSONObject.parseObject(dataJson); + String dataProcessId = parseObject.getString(Constants.DATAPROCESSID); + String appCode = parseObject.getString(Constants.APPCODE); + Map result = new HashMap(16); + int code = 1; + String message = "删除成功"; + try { + if( parseObject.containsKey(Constants.PARENTID) && parseObject.get(Constants.PARENTID) != null ){ + //删除调回索引中的数据 + log.info("扩散型数据删除----"); + List parentIds = (List) parseObject.get(Constants.PARENTID); + String deleteResponse = ElasticsearchUtil.deleteDataByconditionNoVersion(Constants.PARENTID + Constants.POINT_KEYWORD, parentIds.get(parentIds.size()-1), nodes, username, password, indexName,Constants.TERM); + JSONObject deleteObject = JSONObject.parseObject(deleteResponse); + if(deleteObject.containsKey(Constants.ERROR)){ + message = "fial"; + code = 40001; + } + //删除调出索引中数据 + String searchResultStr = ElasticsearchUtil.getDocumentBySingleCondition(Constants.APPCODE, appCode, nodes, username, password, indexName); + JSONObject searchResult = JSONObject.parseObject(searchResultStr); + Map hits = (Map) searchResult.get(Constants.HITS); + List> hitList = (List>) hits.get(Constants.HITS); + if (hitList != null && hitList.size() <= 1) { + if(parseObject.containsKey(Constants.DISPATCHID)){ + //获取调出id --调出数据删除不考虑version问题 + String dispatchId = (String) parseObject.get(Constants.DISPATCHID); + ElasticsearchUtil.deleteDataBycondition(Constants.DATAID, dispatchId, nodes, username, password, dispatchIndexName,"-1"); + } + } + }else{ + //无扩散型应用数据删除 + String deleteResponse = ElasticsearchUtil.deleteDataByconditionNoVersion(Constants.DATAPROCESSID, dataProcessId, nodes, username, password, indexName,Constants.MATCH); + JSONObject deleteObject = JSONObject.parseObject(deleteResponse); + if(deleteObject.containsKey(Constants.ERROR)){ + message = "fial"; + code = 40001; + } + if(parseObject.containsKey(Constants.DISPATCHID)){ + //获取调出id --调出数据删除不考虑version问题 + String dispatchId = (String) parseObject.get(Constants.DISPATCHID); + ElasticsearchUtil.deleteDataBycondition(Constants.DATAID, dispatchId, nodes, username, password, dispatchIndexName,"-1"); + } + } + } catch (Exception e) { + log.error("删除失败-------{}", dataProcessId, e); + code = -1; + } + result.put(Constants.CODE, code); + result.put(Constants.MESSAGE, message); + return JSONObject.toJSONString(result); + } + + @Override + public String retry(String dataJson) { + JSONObject parseObject = JSONObject.parseObject(dataJson); + String appCode = (String) parseObject.get(Constants.APP_CODE); + String conditionField = Constants.APPCODE; + Map result = new HashMap(16); + int code = 0; + String message = "未找到重试节点"; + try { + String searchResultStr = ElasticsearchUtil.getDocumentBySingleCondition(conditionField, appCode, nodes, username, password, indexName); + JSONObject searchResult = JSONObject.parseObject(searchResultStr); + Map hits = (Map) searchResult.get(Constants.HITS); + List> hitList = (List>) hits.get(Constants.HITS); + if (hitList != null && hitList.size() > 0) { + for (Map map : hitList) { + Map esSource = (Map) map.get(Constants.ES_SOURCE); + Integer status = (Integer) esSource.get(Constants.STATUS); + //异常的重试 + if (status != null && status == 2) { + //获取单条完整场景流程 + Map data = new HashMap(16); + String dataProcessId = (String) esSource.get(Constants.DATAPROCESSID); + conditionField = Constants.DATAPROCESSID; + String singleResultStr = ElasticsearchUtil.getDocumentBySingleCondition(conditionField, dataProcessId, nodes, username, password, indexName); + JSONObject singleResult = JSONObject.parseObject(singleResultStr); + Map singleHits = (Map) singleResult.get(Constants.HITS); + List> singleHitList = (List>) singleHits.get(Constants.HITS); + for (Map item : singleHitList) { + Map singleEsSource = (Map) item.get(Constants.ES_SOURCE); + //结果集 data组装 + if (!data.containsKey(singleEsSource.get(Constants.APPCODE))) { + String resultStr = (String) singleEsSource.get(Constants.REQUEST_RESULT); + data.put((String) singleEsSource.get(Constants.APPCODE), resultStr); + } + //组装结果集 --加到调出列中，从而触发当前节点的调用 + if (appCode.equals(singleEsSource.get(Constants.APPCODE))) { + //构建任务 + Map task = JSONObject.parseObject(stringRedisTemplate.opsForValue().get(singleEsSource.get(Constants.SCENESID) + Constants.UNDERLINE + appCode)); + Map newTask = new HashMap(32); + for (Entry entry : task.entrySet()) { + newTask.put(entry.getKey(), entry.getValue()); + } + newTask.put(Constants.DATAID, singleEsSource.get(Constants.DATAID)); + newTask.put(Constants.DATAPROCESSID, singleEsSource.get(Constants.DATAPROCESSID)); + newTask.put(Constants.COVER, true); + newTask.put(Constants.BUSINESSKEY, singleEsSource.get(Constants.BUSINESSKEY)); + if(singleEsSource.containsKey(Constants.DISPATCHID)){ + newTask.put(Constants.DISPATCHID, singleEsSource.get(Constants.DISPATCHID)); + } + //data补充 + newTask.put(Constants.DATA, data); + //newTask或string直接写入队列 + ConfigCache.putQueue(ConfigCache.taskQueue, newTask); + code = 1; + message = "重试任务成功下发"; + log.info("重试任务成功下发，app_code:{},dataId:{}", appCode, singleEsSource.get(Constants.DATAID)); + } + } + } + } + } + } catch (IOException e) { + log.error("重试失败-------{}", dataJson, e); + code = -1; + } + result.put(Constants.CODE, code); + result.put(Constants.MESSAGE, message); + return JSONObject.toJSONString(result); + } + + @Override + public String scenesDelete(String dataJson) { + Map response = new HashMap(16); + int code = 200; + String message = "success"; + try { + Map params = JSONObject.parseObject(dataJson); + Integer scenesId = Integer.valueOf(params.get(Constants.SCENES_ID).toString()); + //调回数据删除 + String deleteResponse = ElasticsearchUtil.deleteDataByconditionNoVersion(Constants.SCENESID, scenesId, nodes, username, password, indexName,Constants.MATCH); + JSONObject parseObject = JSONObject.parseObject(deleteResponse); + if(parseObject.containsKey(Constants.ERROR)){ + message = "fial"; + code = 40001; + }else{ + Map zkResult = new HashMap(16); + String scenesVersion = stringRedisTemplate.opsForValue().get(Constants.VERSION + Constants.UNDERLINE + scenesId); + zkResult.put("scenes_id", scenesId); + zkResult.put("operation", Constants.DELETE); + zkResult.put(Constants.VERSION, scenesVersion); + String value = JSONObject.toJSONString(zkResult); + //删除通知 + curatorFramework.setData().forPath(publishNode, value.getBytes()); + } + }catch (Exception e) { + // TODO: handle exception + log.error("删除未知异常：",e); + message = "fial"; + code = 40002; + } + response.put(Constants.CODE, code); + response.put(Constants.MESSAGE, message); + + return JSONObject.toJSONString(response); + } + + +} diff --git a/src/main/java/com/bfd/analyze/test/CreateElasticsearchIndex.java b/src/main/java/com/bfd/analyze/test/CreateElasticsearchIndex.java new file mode 100644 index 0000000..8f76ad7 --- /dev/null +++ b/src/main/java/com/bfd/analyze/test/CreateElasticsearchIndex.java @@ -0,0 +1,79 @@ +package com.bfd.analyze.test; + +import java.util.Base64; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +/** + * @author jian.mao + * @date 2023年11月3日 + * @description + */ +public class CreateElasticsearchIndex { + public static void main(String[] args) { + // Elasticsearch的主机和端口 + HttpHost elasticsearchHost = new HttpHost("172.16.12.55", 9200); + + // 用户名和密码 + String username = "elastic"; + String password = "bfd123"; + + // Elasticsearch索引名称 + String indexName = "analyze_00001"; + + // 创建一个 HttpClient + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + // 创建一个 HTTP PUT 请求用于创建索引// Elasticsearch主机和索引名称 + HttpPut httpPut = new HttpPut("http://172.16.12.55:9200/" + indexName); + + // 设置基本认证 + httpPut.addHeader("Authorization", "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes())); + + // 设置请求体，定义索引的映射和设置 + String indexDefinition = "{" + + " \"mappings\": {" + + " \"properties\": {" + + " \"created\": {\"type\": \"keyword\"}," + + " \"lastEdit\": {\"type\": \"keyword\"}," + + " \"appId\": {\"type\": \"integer\"}," + + " \"transferId\": {\"type\": \"integer\"}," + + " \"appName\": {\"type\": \"keyword\"}," + + " \"appCode\": {\"type\": \"keyword\"}," + + " \"appDescribe\": {\"type\": \"text\"}," + + " \"module\": {\"type\": \"keyword\"}," + + " \"result\": {\"type\": \"text\"}," + + " \"nextAppId\": {\"type\": \"nested\"}," + + " \"startTag\": {\"type\": \"boolean\"}," + + " \"scenesId\": {\"type\": \"integer\"}," + + " \"businessKey\": {\"type\": \"keyword\"}" + + " }" + + " }" + + "}"; + + httpPut.setEntity(new StringEntity(indexDefinition, ContentType.APPLICATION_JSON)); + + // 发送请求并获取响应 + HttpResponse response = httpClient.execute(elasticsearchHost, httpPut); + + // 处理响应 + int statusCode = response.getStatusLine().getStatusCode(); + String responseBody = EntityUtils.toString(response.getEntity()); + int code = 200; + if (statusCode == code) { + System.out.println("索引创建成功"); + } else { + System.out.println("索引创建失败：" + responseBody); + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/com/bfd/analyze/test/DeleteElasticsearchIndex.java b/src/main/java/com/bfd/analyze/test/DeleteElasticsearchIndex.java new file mode 100644 index 0000000..5f232d3 --- /dev/null +++ b/src/main/java/com/bfd/analyze/test/DeleteElasticsearchIndex.java @@ -0,0 +1,58 @@ +package com.bfd.analyze.test; + +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.impl.client.BasicCredentialsProvider; + +/** + * @author jian.mao + * @date 2023年11月3日 + * @description + */ +public class DeleteElasticsearchIndex { + public static void main(String[] args) { + // Elasticsearch的主机和端口 + HttpHost elasticsearchHost = new HttpHost("172.16.12.55", 9200); + + // Elasticsearch索引名称 + String indexName = "analyze_00001"; + + // 用户名和密码 + String username = "elastic"; + String password = "bfd123"; + + // 创建凭据提供者 + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + + // 创建一个 HttpClient，设置凭据提供者 + try (CloseableHttpClient httpClient = HttpClients.custom() + .setDefaultCredentialsProvider(credentialsProvider) + .build()) { + // 创建一个 HTTP DELETE 请求用于删除索引 + HttpDelete httpDelete = new HttpDelete("http://172.16.12.55:9200/" + indexName); + + // 发送请求并获取响应 + HttpResponse response = httpClient.execute(elasticsearchHost, httpDelete); + + // 处理响应 + int statusCode = response.getStatusLine().getStatusCode(); + String responseBody = EntityUtils.toString(response.getEntity()); + int code = 200; + if (statusCode == code) { + System.out.println("索引删除成功"); + } else { + System.out.println("索引删除失败：" + responseBody); + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/com/bfd/analyze/test/HttpClientDeleteExample.java b/src/main/java/com/bfd/analyze/test/HttpClientDeleteExample.java new file mode 100644 index 0000000..b309134 --- /dev/null +++ b/src/main/java/com/bfd/analyze/test/HttpClientDeleteExample.java @@ -0,0 +1,59 @@ +package com.bfd.analyze.test; + +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import java.io.IOException; + +/** + * @author jian.mao + * @date 2023年11月9日 + * @description + */ +public class HttpClientDeleteExample { + public static void main(String[] args) { + // 创建 HttpClient 连接池 + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); + connManager.setMaxTotal(100); + + // 设置 Elasticsearch 集群地址 + HttpHost[] hosts = { + new HttpHost("172.16.12.55", 9200, "http"), + new HttpHost("172.16.12.56", 9200, "http"), + new HttpHost("172.16.12.57", 9200, "http") + }; + + // 设置用户凭证 + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials("elastic", "bfd123")); + + // 创建 HttpClient 实例并配置连接信息 + CloseableHttpClient httpClient = HttpClients.custom() + .setConnectionManager(connManager) + .setDefaultCredentialsProvider(credentialsProvider) + .build(); + + // 构造删除文档请求 + HttpDelete deleteRequest = new HttpDelete("http://172.16.12.55:9200/analyze_00001/_doc/1"); + deleteRequest.setHeader("Content-Type", "application/json"); + + try { + // 执行删除文档请求 + httpClient.execute(deleteRequest); + + // 关闭 HttpClient 连接 + httpClient.close(); + + System.out.println("删除文档成功"); + } catch (IOException e) { + System.out.println("删除文档出错: " + e.getMessage()); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/bfd/analyze/test/IndexDataWithHttpClient.java b/src/main/java/com/bfd/analyze/test/IndexDataWithHttpClient.java new file mode 100644 index 0000000..8675ad7 --- /dev/null +++ b/src/main/java/com/bfd/analyze/test/IndexDataWithHttpClient.java @@ -0,0 +1,81 @@ +package com.bfd.analyze.test; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +/** + * @author jian.mao + * @date 2023年11月3日 + * @description + */ +public class IndexDataWithHttpClient { + public static void main(String[] args) { + // Elasticsearch索引名称 + String indexName = "analyze_00001"; + + // 创建凭据提供者 + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "bfd123")); + + // 创建一个 HttpClient，设置凭据提供者 + CloseableHttpClient httpClient = HttpClients.custom() + .setDefaultCredentialsProvider(credentialsProvider) + .build(); + try { + // 创建一个 HTTP POST 请求用于写入数据到索引 + // Elasticsearch主机、索引名称和文档ID + HttpPost httpPost = new HttpPost("http://172.16.12.55:9200/" + indexName + "/_doc/1"); + + // 设置请求体，包含要写入的文档数据 + Map document = new HashMap(16); + document.put("created", "2023-11-02"); + document.put("lastEdit", "2023-11-02"); + document.put("appId", 1); + document.put("transferId", 2); + document.put("appName", "Youtube采集"); + document.put("appCode", "ABC1233"); + document.put("appDescribe", "Youtube采集"); + document.put("module", "1"); + document.put("result", "test"); + document.put("nextAppId", new JSONArray()); + document.put("startTag", true); + document.put("scenesId", 1); + document.put("businessKey", "B321321y"); + + StringEntity entity = new StringEntity(JSONObject.toJSONString(document), ContentType.APPLICATION_JSON); + httpPost.setEntity(entity); + + // 发送请求并获取响应 + HttpResponse response = httpClient.execute( httpPost); + + // 处理响应 + int statusCode = response.getStatusLine().getStatusCode(); + String responseBody = EntityUtils.toString(response.getEntity()); + int code = 201; + if (statusCode == code) { + System.out.println("文档写入成功"); + } else { + System.out.println("文档写入失败：" + responseBody); + } + httpClient.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/com/bfd/analyze/utils/DateUtil.java b/src/main/java/com/bfd/analyze/utils/DateUtil.java new file mode 100644 index 0000000..e4d968e --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/DateUtil.java @@ -0,0 +1,177 @@ +package com.bfd.analyze.utils; + + +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Date; + +import lombok.extern.slf4j.Slf4j; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; + +/** + * 日期工具类 + * + * @author jian.mao + * @date 2022年11月15日 + * @description + */ +@Slf4j +public class DateUtil { + + /** + * @return + */ + public static String getTimeStrForNow() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH"); + return sdf.format(new Date()); + } + + + public static String getTimeStrForDay(long time) { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); + + return sdf.format(new Date(time * 1000)); + } + + public static String getTimeStrForDay() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); + + return sdf.format(new Date()); + } + + + public static String getDateTime() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String time = sdf.format(new Date()); + return time; + } + + public static String getDateTime(Long timestap) { + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String time = sdf.format(new Date(timestap)); + return time; + } + + public static String getDate(Long timestap) { + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + String time = sdf.format(new Date(timestap)); + return time; + } + + public static String getDateTimeForMonth() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM"); + String time = sdf.format(new Date()); + return time; + } + + /** + * 休眠 + * + * @param millis 毫秒 + */ + public static void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + /** + * 1. @Description:时间戳转时间 + * 2. @Author: ying.zhao + * 3. @Date: 2023/3/28 + */ + + public static String timestampToDate(String time) { + int thirteen = 13; + int ten = 10; + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); +// if (time.length() == thirteen) { + if (time.length() > ten) { + return sdf.format(new Date(Long.parseLong(time))); + } else { + return sdf.format(new Date(Integer.parseInt(time) * 1000L)); + } + } + + public static String parseCreated(String jsonTime){ + String formattedDateTime = getDateTime(); + try { + // 使用fastjson解析JSON数据 + JSONObject jsonObject = JSON.parseObject(jsonTime); + // 获取日期和时间的值 + JSONObject dateObject = jsonObject.getJSONObject("date"); + int day = dateObject.getIntValue("day"); + int month = dateObject.getIntValue("month"); + int year = dateObject.getIntValue("year"); + + JSONObject timeObject = jsonObject.getJSONObject("time"); + int hour = timeObject.getIntValue("hour"); + int minute = timeObject.getIntValue("minute"); + int second = timeObject.getIntValue("second"); + + // 创建LocalDateTime对象 + LocalDateTime dateTime = LocalDateTime.of(year, month, day, hour, minute, second); + + // 定义日期时间格式化器 + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + // 格式化日期时间 + formattedDateTime = dateTime.format(formatter); + } catch (Exception e) { + log.info("日期转换失败：{}",e); + } + return formattedDateTime; + } + + /** + * 字符串转换日期 + * @param format + * @param date + * @return + */ + public static Date strToDate(String format,String date){ + SimpleDateFormat sdf = new SimpleDateFormat(format); + if (date == null || date.equals("")){ + return new Date(); + }else{ + Date ru = null; + try { + ru = sdf.parse(date); + } catch (ParseException e) { + e.printStackTrace(); + } + return ru; + } + } + /** + * 日期格式话 + * @param format 日期格式 + * @param dater 要转换的日期,默认当前时间 + * @return + */ + public static String FormatDate(String format,Date date){ + String fromatDate = null; + SimpleDateFormat sdf = new SimpleDateFormat(format); + if (date == null){ + fromatDate = sdf.format(new Date()); + }else{ + fromatDate = sdf.format(date); + } + return fromatDate; + } + public static void main(String[] args) { + String time = timestampToDate("955814400000"); + System.out.println(time); + } +} diff --git a/src/main/java/com/bfd/analyze/utils/DownLoadUtil.java b/src/main/java/com/bfd/analyze/utils/DownLoadUtil.java new file mode 100644 index 0000000..7266aed --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/DownLoadUtil.java @@ -0,0 +1,912 @@ +package com.bfd.analyze.utils; + +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.StatusLine; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.AuthCache; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.HttpClient; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.LayeredConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.BasicAuthCache; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.LaxRedirectStrategy; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.bfd.analyze.entity.Constants; + + + + +/** + * @author jian.mao + * @date 2023年12月7日 + * @description + */ +public class DownLoadUtil { + + private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"; + private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class); + /** 代理服务器(产品官网 www.16yun.cn) **/ + final static String PROXYHOST = "u270.40.tp.16yun.cn"; + final static Integer PROXYPORT = 6448; + /** 代理验证信息 **/ + final static String PROXYUSER = "16HFBVJC"; + final static String PROXYPASS = "897944"; + + private static PoolingHttpClientConnectionManager cm = null; + private static HttpRequestRetryHandler httpRequestRetryHandler = null; + private static HttpHost proxy = null; + + private static CredentialsProvider credsProvider = null; + private static RequestConfig reqConfig = null; + + static { + ConnectionSocketFactory plainsf = PlainConnectionSocketFactory + .getSocketFactory(); + LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory + .getSocketFactory(); + + Registry registry = RegistryBuilder.create().register("http", plainsf) + .register("https", sslsf).build(); + + cm = new PoolingHttpClientConnectionManager(registry); + cm.setMaxTotal(20); + cm.setDefaultMaxPerRoute(5); + + proxy = new HttpHost(PROXYHOST, PROXYPORT, "https"); + + credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(PROXYUSER, PROXYPASS)); + + reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000) + .setConnectTimeout(5000).setSocketTimeout(5000) + .setExpectContinueEnabled(false) + .setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build(); + } + + /** + * 模拟客户端get请求 + * + * @param url + * 模拟请求得url + * @param headers + * 头部信息，没有可以不传 + * @return + */ + @SafeVarargs + public static String proxyDoGet(String url, Map... headers) { + // 设置超时时间 + int timeout = 30; + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) + .setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); + AuthCache authCache = new BasicAuthCache(); + authCache.put(proxy, new BasicScheme()); + HttpClientContext localContext = HttpClientContext.create(); + localContext.setAuthCache(authCache); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + CloseableHttpClient httpClient = httpBuilder + .setDefaultSocketConfig(socketConfig) + .setDefaultRequestConfig(config) + .setDefaultCredentialsProvider(credsProvider).build(); + HttpGet httpGet = new HttpGet(url); + httpGet.setConfig(reqConfig); + if (headers != null && headers.length > 0) { + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key, tempHeaders.get(key).toString()); + } + } else { + httpGet.setHeader("Accept", + "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + } + CloseableHttpResponse response = null; + String html = ""; + int notFundCode = 404; + int successCode = 200; + try { + response = httpClient.execute(httpGet, localContext); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + if (statusLine.getStatusCode() == successCode) { + if (responseEntity != null) { + html = EntityUtils.toString(responseEntity, "utf-8"); + System.out.println("响应内容长度为:" + + responseEntity.getContentLength()); + // 下载结果为空不正常 + if (html.equals(Constants.EMPTY)) { + html = "Download failed error is:reslut is null"; + } + } + } else if (statusLine.getStatusCode() == notFundCode) { + html = "

页面404，正常结束请求即可

"; + } else { + throw new Exception("请求错误，code码为：" + statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:reslut is null"; + }finally{ + try { + response.close(); + httpClient.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + return html; + + } + + + public static String httpsslProxyGet(String url, Map... headers) throws Exception { + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connManager.setMaxTotal(50); + connManager.setDefaultMaxPerRoute(10); + HttpClients.custom().setConnectionManager(connManager); + // 设置超时时间 + int timeout = 30; + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) + .setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); + AuthCache authCache = new BasicAuthCache(); + authCache.put(proxy, new BasicScheme()); + HttpClientContext localContext = HttpClientContext.create(); + localContext.setAuthCache(authCache); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + CloseableHttpClient httpClient = httpBuilder + .setConnectionManager(connManager) + .setDefaultSocketConfig(socketConfig) + .setDefaultRequestConfig(config) + .setDefaultCredentialsProvider(credsProvider).build(); + HttpGet httpGet = new HttpGet(url); + httpGet.setConfig(reqConfig); + if (headers != null && headers.length > 0) { + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key, tempHeaders.get(key).toString()); + } + } else { + httpGet.setHeader("Accept", + "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + } + CloseableHttpResponse response = null; + String html = ""; + int notFundCode = 404; + int successCode = 200; + try { + response = httpClient.execute(httpGet, localContext); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + if (statusLine.getStatusCode() == successCode) { + if (responseEntity != null) { + html = EntityUtils.toString(responseEntity, "utf-8"); + System.out.println("响应内容长度为:" + + responseEntity.getContentLength()); + // 下载结果为空不正常 + if (html.equals(Constants.EMPTY)) { + html = "Download failed error is:reslut is null"; + } + } + } else if (statusLine.getStatusCode() == notFundCode) { + html = "

页面404，正常结束请求即可

"; + } else { + throw new Exception("请求错误，code码为：" + statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:reslut is null"; + }finally{ + try { + response.close(); + httpClient.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + return html; + + } + + + /** + * json参数方式POST提交 + * @param url + * @param params + * @return + */ + public static String doPost(String url, String params){ + String strResult = ""; + //设置超时时间 + int timeout =60; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); +// AuthCache authCache = new BasicAuthCache(); +// authCache.put(proxy, new BasicScheme()); +// HttpClientContext localContext = HttpClientContext.create(); +// localContext.setAuthCache(authCache); + // 1. 获取默认的client实例 + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); + HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); +// HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) +// .setDefaultCredentialsProvider(credsProvider).build(); + // 2. 创建httppost实例 + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); + HttpResponse resp = null; + try { + httpPost.setEntity(new StringEntity(params,"utf-8")); + resp = client.execute(httpPost); +// resp = client.execute(httpPost,localContext); + StatusLine statusLine = resp.getStatusLine(); + System.out.println("响应状态为:" + resp.getStatusLine()); + int successCode = 200; + int errorCode = 404; + if(statusLine.getStatusCode() == successCode){ + // 7. 获取响应entity + HttpEntity respEntity = resp.getEntity(); + strResult = EntityUtils.toString(respEntity, "UTF-8"); + if(strResult.equals("")){ + strResult = "Download failed error is:reslut is null"; + } + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + return strResult; + } + public static String httpPost(String url,String params) { + String html=""; + html = doPost(url,params); + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(5000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + html = doPost(url,params); + } + return html; + } + /** + * 绕过验证 + * + * @return + * @throws NoSuchAlgorithmException + * @throws KeyManagementException + */ + public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException { + SSLContext sc = SSLContext.getInstance("SSLv3"); + + // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法 + X509TrustManager trustManager = new X509TrustManager() { + @Override + public void checkClientTrusted( + java.security.cert.X509Certificate[] paramArrayOfX509Certificate, + String paramString) throws CertificateException { + } + + @Override + public void checkServerTrusted( + java.security.cert.X509Certificate[] paramArrayOfX509Certificate, + String paramString) throws CertificateException { + } + + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + }; + + sc.init(null, new TrustManager[] { trustManager }, null); + return sc; + } + /** + * 模拟请求 + * + * @param url 资源地址 + * @param map 参数列表 + * @param encoding 编码 + * @return + * @throws NoSuchAlgorithmException + * @throws KeyManagementException + * @throws IOException + * @throws ClientProtocolException + */ + public static String httpsslGet(String url,Map ... headers) { + String html=""; + CloseableHttpClient client = null; + HttpEntity responseEntity = null; + CloseableHttpResponse response = null; + try { + log.debug("DownLoadUtil------------->设置下载相关信息， start...."); + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connManager.setMaxTotal(50); + connManager.setDefaultMaxPerRoute(10); + HttpClients.custom().setConnectionManager(connManager); + //设置超时时间 + int timeout = 30; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + // 设置重定向策略 + LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy(); + //创建自定义的httpclient对象 + client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build(); +// CloseableHttpClient client = HttpClients.createDefault(); + + HttpGet httpGet = new HttpGet(url); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); + } + log.debug("DownLoadUtil------------->设置下载相关信息， end...."); + try { + log.debug("DownLoadUtil------------->下载执行，start...."); + httpGet.setConfig(config); + response = client.execute(httpGet); + log.debug("DownLoadUtil------------->下载执行，end...."); + // 从响应模型中获取响应实体 + StatusLine statusLine = response.getStatusLine(); + log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is start ...."); + System.out.println("响应状态为:" + response.getStatusLine()); + responseEntity = response.getEntity(); + log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is end ...."); + int successCode = 200; + int errorCode = 404; + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + System.out.println("响应内容长度为:" + responseEntity.getContentLength()); + } + }else if(statusLine.getStatusCode() == errorCode){ + html = "

页面404，正常结束请求即可

"; + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + }finally{ + try { + responseEntity.getContent().close(); + response.close(); + client.close(); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + return html; + } + + public static String httpSSLGet(String url,Map ... headers) { + String html=""; + html = httpsslGet(url,headers); + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(30000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + html = httpsslGet(url,headers); + } + return html; + } + public static String doPostFrom(String url,Map param,Map ... headers){ + //设置超时时间 + int timeout = 15; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); +// AuthCache authCache = new BasicAuthCache(); +// authCache.put(proxy, new BasicScheme()); +// HttpClientContext localContext = HttpClientContext.create(); +// localContext.setAuthCache(authCache); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); +// HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) +// .setDefaultCredentialsProvider(credsProvider).build(); + HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"); + httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); + httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"); +// httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html"); + } + // 创建请求参数 + List list = new LinkedList<>(); + for (String key : param.keySet()) { + BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); + list.add(param1); + } + // 使用URL实体转换工具 + String html=""; + try { + UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); + httpPost.setEntity(entityParam); + HttpResponse response = httpClient.execute(httpPost); +// HttpResponse response = httpClient.execute(httpPost,localContext); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + int successCode = 200; + int errorCode = 404; + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + } + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + + return html; + + } + public static String httpPostForm(String url,Map params,Map ... headers) { + String html=""; + html = doPostFrom(url,params); + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(5000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + html = doPostFrom(url,params,headers); + } + return html; + } + + public static String dosslPost(String url,String params,Map ... headers) { + String html=""; + CloseableHttpClient client = null; + HttpEntity responseEntity = null; + CloseableHttpResponse response = null; + try { + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + HttpClients.custom().setConnectionManager(connManager); + //设置超时时间 + int timeout = 5; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + //创建自定义的httpclient对象 + client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); +// CloseableHttpClient client = HttpClients.createDefault(); + // 2. 创建httppost实例 + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); + } + + try { + httpPost.setEntity(new StringEntity(params,"utf-8")); + response = client.execute(httpPost); + // 从响应模型中获取响应实体 + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + responseEntity = response.getEntity(); + int successCode = 200; + int errorCode = 404; + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + System.out.println("响应内容长度为:" + responseEntity.getContentLength()); + } + }else if(statusLine.getStatusCode() == errorCode){ + html = "

页面404，正常结束请求即可

"; + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + }finally{ + try { + responseEntity.getContent().close(); + response.close(); + client.close(); + } catch (UnsupportedOperationException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + + return html; + } + public static String dosslPostForm(String url,Map param,Map ... headers) { + String html=""; + try { + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + HttpClients.custom().setConnectionManager(connManager); + //设置超时时间 + int timeout = 5; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + //创建自定义的httpclient对象 + CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); +// CloseableHttpClient client = HttpClients.createDefault(); + // 2. 创建httppost实例 + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); + httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); + } + + // 创建请求参数 + List list = new LinkedList<>(); + for (String key : param.keySet()) { + BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); + list.add(param1); + } + // 使用URL实体转换工具 + try { + UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); + httpPost.setEntity(entityParam); + HttpResponse response = client.execute(httpPost); +// HttpResponse response = httpClient.execute(httpPost,localContext); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + int successCode = 200; + int errorCode = 404; + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + } + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + + + return html; + } + public static String httpSSLPostForm(String url,Map params,Map ...headers) { + String html=""; + try { + html = dosslPostForm(url,params,headers); + } catch (Exception e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(30000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + try { + html = dosslPostForm(url,params,headers); + } catch (Exception e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + } + return html; + } + public static String httpSSLPost(String url,String params,Map ...headers) { + String html=""; + try { + html = dosslPost(url,params,headers); + } catch (Throwable e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(30000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + try { + html = dosslPost(url,params,headers); + } catch (Throwable e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + } + return html; + } + + /** + * 模拟客户端get请求 + * @param url 模拟请求得url + * @param headers 头部信息，没有可以不传 + * @return + */ + public static String doGet(String url,Map ... headers){ + //设置超时时间 + int timeout = 15; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); + HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); + HttpGet httpGet = new HttpGet(url); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + } + String html=""; + try { + HttpResponse response = httpClient.execute(httpGet); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + int successCode = 200; + int errorCode = 404; + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + if(html.equals("")){ + html = "Download failed error is:reslut is null"; + } + } + }else if(statusLine.getStatusCode() == errorCode){ + html = "

页面404，正常结束请求即可

"; + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + return html; + + } +// public static void main(String[] args) throws Exception { +// Map headers = new HashMap(); +// headers.put("accept", "application/json, text/plain, */*"); +// headers.put("authority", "mp.weixin.qq.com"); +// headers.put("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"); +// headers.put("accept-language", "zh-CN,zh;q=0.9,en;q=0.8"); +// headers.put("user-agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"); +// String url = "https://mp.weixin.qq.com/s?__biz=MzU5NDgzNTg2OQ==&mid=2247528413&idx=2&sn=cc79680d6f6cc160e7aa3f5d2101f2ae"; +// String html = httpsslProxyGet(url, headers); +// System.out.println(html); +// } +} diff --git a/src/main/java/com/bfd/analyze/utils/ElasticsearchUtil.java b/src/main/java/com/bfd/analyze/utils/ElasticsearchUtil.java new file mode 100644 index 0000000..c65b327 --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/ElasticsearchUtil.java @@ -0,0 +1,408 @@ +package com.bfd.analyze.utils; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.analyze.entity.Constants; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +/** + * es操作工具类 + * + * @author jian.mao + * @date 2024年6月6日 + * @description + */ +@Slf4j +public class ElasticsearchUtil { + + /** + * es根据条件更新 + * + * @param index + * 索引 + * @param username + * 用户名 + * @param password + * 密码 + * @param field + * 更新字段名称 + * @param fieldValue + * 更新字段值 + * @param conditionField + * 条件字段名称 + * @param conditionValue + * 条件值 + * @return + * @throws IOException + */ + public static String updateFieldByCondition(String index, String username, String password, String field, + Object fieldValue, String conditionField, Object conditionValue,String nodes) + throws IOException { + //只写入一个节点 + String node = nodes.split(",")[0]; + // 构造请求 URL + StringBuffer url = new StringBuffer("http://"); + url.append(node); + url.append("/"); + url.append(index); + url.append("/_update_by_query?refresh=true"); + // Script part + Map params = new HashMap<>(16); + params.put("value", fieldValue); + Map script = new HashMap<>(16); + script.put("source", + String.format("ctx._source.%s = params.value", field)); + script.put("lang", "painless"); + script.put("params", params); + // Query part + Map match = new HashMap<>(16); + match.put(conditionField, conditionValue); + Map query = new HashMap<>(16); + query.put("match", match); + // Combine script and query + Map requestBody = new HashMap<>(16); + requestBody.put("script", script); + requestBody.put("query", query); + // Convert Map to JSON string + String jsonBody = JSONObject.toJSONString(requestBody); + return sendPostRequest(url.toString(), jsonBody, username, password); + } + + /** + * @param index 索引 + * @param id 主键 + * @param username 用户名 + * @param password 密码 + * @param field 更新字段 + * @param fieldValue 字段值 + * @param nodes 节点信息 + * @return + * @throws IOException + */ + public static String updateFieldById(String index, String id, String username, String password, String field, + Object fieldValue, String nodes) throws IOException { + // 只写入一个节点 + String node = nodes.split(",")[0]; + // 构造请求 URL + String url = String.format("http://%s/%s/_update/%s", node, index, id); + + // 构造请求脚本 + Map params = new HashMap(16); + params.put("value", fieldValue); + Map scriptContent = new HashMap(16); + scriptContent.put("source", String.format("ctx._source.%s = params.value", field)); + scriptContent.put("lang", "painless"); + scriptContent.put("params", params); + + // 将 Script 内容嵌套在脚本对象中 + Map script = new HashMap(16); + script.put("script", scriptContent); + + // 将 Map 转换为 JSON 字符串 + String jsonBody = JSONObject.toJSONString(script); + + + // 发送 POST 请求 + return sendPostRequest(url, jsonBody, username, password); + } + /** + * rest接口请求es + * + * @param url + * @param jsonBody + * @param username + * @param password + * @return + * @throws IOException + */ + private static String sendPostRequest(String url, String jsonBody, + String username, String password) throws IOException { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost httpPost = new HttpPost(url); + // Set the content type to JSON + httpPost.setHeader("Content-Type", "application/json"); + // Set the basic authentication header + String auth = username + ":" + password; + byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes()); + String authHeader = "Basic " + new String(encodedAuth); + httpPost.setHeader("Authorization", authHeader); + + // Set the JSON body + StringEntity entity = new StringEntity(jsonBody); + httpPost.setEntity(entity); + + // Execute the request + return EntityUtils.toString(httpClient.execute(httpPost) + .getEntity()); + } + } + + + /** + * 单条件查询 + * @param conditionField + * @param conditionValue + * @param nodes + * @param username + * @param password + * @param indexName + * @return + * @throws IOException + */ + public static String getDocumentBySingleCondition(String conditionField, Object conditionValue,String nodes,String username,String password,String indexName) throws IOException { + //只写入一个节点 + String node = nodes.split(",")[0]; + // 创建凭据提供者 + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + // 创建一个 HttpClient，设置凭据提供者 + try (CloseableHttpClient httpClient = HttpClients.custom() + .setDefaultCredentialsProvider(credentialsProvider) + .build()) { + // 构造请求 URL + StringBuffer host = new StringBuffer("http://"); + host.append(node); + host.append("/"); + host.append(indexName); + host.append("/_search"); + // 构造请求体 + String requestBody = String.format("{ \"query\": { \"match\": { \"%s\": \"%s\" }}}",conditionField, conditionValue); + // 创建 HTTP POST 请求 + HttpPost httpPost = new HttpPost(host.toString()); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setEntity(new StringEntity(requestBody)); + // 发送请求并获取响应 + HttpResponse response = httpClient.execute(httpPost); + // 解析响应 + HttpEntity entity = response.getEntity(); + if (entity != null) { + String responseString = EntityUtils.toString(entity); + log.info("条件：{}，条件查询响应体：{}" ,requestBody,responseString); + return responseString; + } else { + log.warn("条件查询异常"); + return Constants.EMPTY; + } + } + } + + /** + * 查询数据，自定义请求体方法 + * @param nodes + * @param username + * @param password + * @param indexName + * @param requestBody + * @return + * @throws IOException + */ + public static String getDocument(String nodes,String username,String password,String indexName,String requestBody) throws IOException { + //只写入一个节点 + String node = nodes.split(",")[0]; + // 创建凭据提供者 + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + // 创建一个 HttpClient，设置凭据提供者 + try (CloseableHttpClient httpClient = HttpClients.custom() + .setDefaultCredentialsProvider(credentialsProvider) + .build()) { + // 构造请求 URL + StringBuffer host = new StringBuffer("http://"); + host.append(node); + host.append("/"); + host.append(indexName); + host.append("/_search"); + // 构造请求体 + // 创建 HTTP POST 请求 + HttpPost httpPost = new HttpPost(host.toString()); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setEntity(new StringEntity(requestBody)); + // 发送请求并获取响应 + HttpResponse response = httpClient.execute(httpPost); + // 解析响应 + HttpEntity entity = response.getEntity(); + if (entity != null) { + String responseString = EntityUtils.toString(entity); + log.info("条件查询响应体：{}" ,responseString); + return responseString; + } else { + log.warn("条件查询异常"); + return Constants.EMPTY; + } + } + } + + /** + * 条件删除，含version + * @param conditionField + * @param conditionValue + * @param nodes + * @param username + * @param password + * @param indexName + * @param version + * @return + */ + public static String deleteDataBycondition(String conditionField, Object conditionValue,String nodes,String username,String password,String indexName,String version){ + + //只写入一个节点 + String node = nodes.split(",")[0]; + // 创建凭据提供者 + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + // 创建一个 HttpClient，设置凭据提供者 + try (CloseableHttpClient httpClient = HttpClients.custom() + .setDefaultCredentialsProvider(credentialsProvider) + .build()) { + // 构造请求 URL + StringBuffer host = new StringBuffer("http://"); + host.append(node); + host.append("/"); + host.append(indexName); + host.append("/_delete_by_query?refresh=true"); + // 构造请求体 +// String requestBody = String.format("{ \"query\": { \"match\": { \"%s\": \"%s\" }}}",conditionField, conditionValue); + //删除条件 添加version + String requestBody = String.format("{\"query\":{\"bool\":{\"must\":[{\"match\":{\"%s\":\"%s\"}}],\"must_not\":[{\"match\":{\"version\":\"%s\"}}]}}}",conditionField, conditionValue,version); + // 创建 HTTP POST 请求 + HttpPost httpPost = new HttpPost(host.toString()); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setEntity(new StringEntity(requestBody)); + // 发送请求并获取响应 + HttpResponse response = httpClient.execute(httpPost); + // 解析响应 + HttpEntity entity = response.getEntity(); + if (entity != null) { + String responseString = EntityUtils.toString(entity); + log.info("条件：{}，有版本号删除响应体：{}" ,requestBody, responseString); + return responseString; + } else { + log.warn("有版本号删除异常"); + return Constants.EMPTY; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + /** + * 条件删除，不含version + * @param conditionField + * @param conditionValue + * @param nodes + * @param username + * @param password + * @param indexName + * @return + */ + public static String deleteDataByconditionNoVersion(String conditionField, Object conditionValue,String nodes,String username,String password,String indexName,String searchType){ + + //只写入一个节点 + String node = nodes.split(",")[0]; + // 创建凭据提供者 + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + // 创建一个 HttpClient，设置凭据提供者 + try (CloseableHttpClient httpClient = HttpClients.custom() + .setDefaultCredentialsProvider(credentialsProvider) + .build()) { + // 构造请求 URL + StringBuffer host = new StringBuffer("http://"); + host.append(node); + host.append("/"); + host.append(indexName); + host.append("/_delete_by_query?refresh=true"); + // 构造请求体 + String requestBody = String.format("{ \"query\": { \"%s\": { \"%s\": \"%s\" }}}",searchType,conditionField, conditionValue); + // 创建 HTTP POST 请求 + HttpPost httpPost = new HttpPost(host.toString()); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setEntity(new StringEntity(requestBody)); + // 发送请求并获取响应 + HttpResponse response = httpClient.execute(httpPost); + // 解析响应 + HttpEntity entity = response.getEntity(); + if (entity != null) { + String responseString = EntityUtils.toString(entity); + log.info("条件：{}，无版本号删除响应体：{}",requestBody,responseString); + return responseString; + } else { + log.warn("无版本号删除异常"); + return Constants.EMPTY; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * 根据主键删除 + * @param id + * @param nodes + * @param username + * @param password + * @param indexName + * @return + */ + public static String deleteDataById(String id, String nodes, String username, String password, String indexName) { + // 只写入一个节点 + String node = nodes.split(",")[0]; + // 创建凭据提供者 + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + // 创建一个 HttpClient，设置凭据提供者 + try (CloseableHttpClient httpClient = HttpClients.custom() + .setDefaultCredentialsProvider(credentialsProvider) + .build()) { + // 构造请求 URL + StringBuilder host = new StringBuilder("http://"); + host.append(node); + host.append("/"); + host.append(indexName); + host.append("/_doc/"); + host.append(id); + // 创建 HTTP DELETE 请求 + HttpDelete httpDelete = new HttpDelete(host.toString()); + // 发送请求并获取响应 + HttpResponse response = httpClient.execute(httpDelete); + // 解析响应 + HttpEntity entity = response.getEntity(); + if (entity != null) { + String responseString = EntityUtils.toString(entity); + log.info("主键删除响应体：{}",responseString); + return responseString; + } else { + log.warn("主键删除异常"); + return Constants.EMPTY; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static void main(String[] args) { + String id = "fb2163f8-b1ee-4f96-8919-12b45d4a0df1"; + String nodes = "172.18.1.147:9200"; + String username = "elastic"; + String password = "baifendian"; + String indexName = "dispatch_analyze_00001"; + deleteDataById(id, nodes, username, password, indexName); + } +} diff --git a/src/main/java/com/bfd/analyze/utils/EncryptionUtil.java b/src/main/java/com/bfd/analyze/utils/EncryptionUtil.java new file mode 100644 index 0000000..b4a4933 --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/EncryptionUtil.java @@ -0,0 +1,27 @@ +package com.bfd.analyze.utils; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * @author jian.mao + * @date 2023年3月10日 + * @description + */ +public class EncryptionUtil { + public static String md5(String text) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(text.getBytes()); + byte[] bytes = md.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b & 0xff)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + return null; + } + } +} diff --git a/src/main/java/com/bfd/analyze/utils/FileUtil.java b/src/main/java/com/bfd/analyze/utils/FileUtil.java new file mode 100644 index 0000000..c22c62d --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/FileUtil.java @@ -0,0 +1,30 @@ +package com.bfd.analyze.utils; + +import java.io.FileWriter; +import java.io.IOException; + +/** + * 文件工具类 + * @author jian.mao + * @date 2023年7月14日 + * @description + */ +public class FileUtil { + + /** + * 数据写入文件 + * @param Path 文件路径 + * @param result 数据 + * @throws IOException + */ + public static void writeFile(String path,String result){ + try { + FileWriter fw = new FileWriter(path,true); + fw.write(result+"\n"); + fw.flush(); + fw.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/com/bfd/analyze/utils/JsonUtil.java b/src/main/java/com/bfd/analyze/utils/JsonUtil.java new file mode 100644 index 0000000..42b5294 --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/JsonUtil.java @@ -0,0 +1,39 @@ +package com.bfd.analyze.utils; + +import java.util.List; +import java.util.Map; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.analyze.entity.Constants; + +/** + * json工具 + * @author jian.mao + * @date 2023年7月10日 + * @description + */ +public class JsonUtil { + + /** + * 校验字符串是list/map/str + * @param jsonString + * @return + */ + public static String checkJsonType(String jsonString) { + try { + JSONObject.parseObject(jsonString); + return Constants.MAP_TYPE; + } catch (Exception e) { + try { + List list = JSONObject.parseArray(jsonString); + if(list != null && list.size() > 0 ){ + return Constants.LIST_TYPE; + }else{ + throw new Exception(); + } + } catch (Exception ex) { + return Constants.STRING_TYPE; + } + } + } +} diff --git a/src/main/java/com/bfd/analyze/utils/OtherUtils.java b/src/main/java/com/bfd/analyze/utils/OtherUtils.java new file mode 100644 index 0000000..e55f449 --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/OtherUtils.java @@ -0,0 +1,32 @@ +package com.bfd.analyze.utils; + +import java.security.MessageDigest; + +/** + * @author jian.mao + * @date 2023年10月7日 + * @description + */ +public class OtherUtils { + + + + public static String getMd5(String string) { + try { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + byte[] bs = md5.digest(string.getBytes("UTF-8")); + StringBuilder sb = new StringBuilder(40); + for (byte x : bs) { + if ((x & 0xff) >> 4 == 0) { + sb.append("0").append(Integer.toHexString(x & 0xff)); + } else { + sb.append(Integer.toHexString(x & 0xff)); + } + } + return sb.toString(); + } catch (Exception e) { + + return "nceaform" + System.currentTimeMillis(); + } + } +} diff --git a/src/main/java/com/bfd/analyze/utils/PauseTool.java b/src/main/java/com/bfd/analyze/utils/PauseTool.java new file mode 100644 index 0000000..87cb486 --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/PauseTool.java @@ -0,0 +1,69 @@ +package com.bfd.analyze.utils; + +import com.alibaba.fastjson.JSON; +import com.bfd.analyze.entity.Constants; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Set; + +/** + * @author:jinming + * @className:ZookeeperNodeMonitor + * @version:1.0 + * @description: Zookeeper节点监听和Redis初始化工具类 + * @Date:2024/7/2 14:20 + */ +@Component +@Slf4j +public class PauseTool { + + /** + * 本地缓存 + */ + + public static final HashMap CACHE = new HashMap<>(); + + @Autowired + private StringRedisTemplate stringRedisTemplate; + + /** + * 初始化Redis中的version_*键并加载到本地缓存 + */ + public void initializeRedisCache(StringRedisTemplate stringRedisTemplate) { + try { + Set keys = stringRedisTemplate.keys("version_*"); + if (keys != null) { + for (String key : keys) { + String value = stringRedisTemplate.opsForValue().get(key); + if (value != null) { + String sincesId = key.split("_")[1]; + CACHE.put(sincesId.concat("_").concat(value), value); + } + } + } + log.info("当前缓存version信息:{}", JSON.toJSON(CACHE)); + } catch (Exception e) { + log.error("Error initializing Redis cache", e); + } + } + + /** + * 校验 任务是否暂停 + * @param scenesId + * @param version + * @return + */ + public Boolean check(int scenesId,int version){ + String scenesVsersion = stringRedisTemplate.opsForValue().get(Constants.VERSION + Constants.UNDERLINE + scenesId); + int scenes_version = Integer.parseInt(scenesVsersion); + if (scenes_version==version){ + return true; + }else { + return false; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/bfd/analyze/utils/QueueUtil.java b/src/main/java/com/bfd/analyze/utils/QueueUtil.java new file mode 100644 index 0000000..4d3adb0 --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/QueueUtil.java @@ -0,0 +1,107 @@ +package com.bfd.analyze.utils; + +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.LinkedBlockingQueue; + + +/** + * 队列工具类 + * @author jian.mao + * @date 2023年3月22日 + * @description + */ +public class QueueUtil { + /****任务****/ + public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>(); + /****详情任务队列***/ + public static LinkedBlockingDeque> taskDetailQueue = new LinkedBlockingDeque>(); + /****股权出质****/ + public static LinkedBlockingDeque> gqczQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> gqczErrorQueue = new LinkedBlockingDeque>(); + /****动产抵押****/ + public static LinkedBlockingDeque> dcdyQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> dcdyErrorQueue = new LinkedBlockingDeque>(); + /****对外担保****/ + public static LinkedBlockingDeque> dydbQueue = new LinkedBlockingDeque>(); + /**分支机构**/ + public static LinkedBlockingDeque> fzjgQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> fzjgErrorQueue = new LinkedBlockingDeque>(); + /***对外担保***/ + public static LinkedBlockingDeque> dwdbQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> dwdbErrorQueue = new LinkedBlockingDeque>(); + /****网站备案***/ + public static LinkedBlockingDeque> wzbaQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> wzbaErrorQueue = new LinkedBlockingDeque>(); + /****经营异常****/ + public static LinkedBlockingDeque> jyycQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> jyycErrorQueue = new LinkedBlockingDeque>(); + /***欠税公告***/ + public static LinkedBlockingDeque> qsggQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> qsggErrorQueue = new LinkedBlockingDeque>(); + /***工商变更***/ + public static LinkedBlockingDeque> gsbgQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> gsbgErrorQueue = new LinkedBlockingDeque>(); + /****专利信息****/ + public static LinkedBlockingDeque> zlxxQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> zlxxErrorQueue = new LinkedBlockingDeque>(); + /****税收违法****/ + public static LinkedBlockingDeque> sswfQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> sswfErrorQueue = new LinkedBlockingDeque>(); + /****涉诉公告-法律诉讼****/ + public static LinkedBlockingDeque> ssggQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> ssggErrorQueue = new LinkedBlockingDeque>(); + /****疑似实际控股人****/ + public static LinkedBlockingDeque> yssjkgrQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> yssjkgrErrorQueue = new LinkedBlockingDeque>(); + /****舆情信息****/ + public static LinkedBlockingDeque> yqxxQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> yqxxErrorQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> yqxxEveryDayQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> yqxxEveryDayDetailQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> yqxxEveryDayErrorQueue = new LinkedBlockingDeque>(); + /****司法协助****/ + public static LinkedBlockingDeque> sfxzQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> sfxzErrorQueue = new LinkedBlockingDeque>(); + /****开庭公告****/ + public static LinkedBlockingDeque> ktggQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> ktggErrorQueue = new LinkedBlockingDeque>(); + /****被执行人****/ + public static LinkedBlockingDeque> bzxrQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> bzxrErrorQueue = new LinkedBlockingDeque>(); + /****失信被执行人****/ + public static LinkedBlockingDeque> sxbzxrQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> sxbzxrErrorQueue = new LinkedBlockingDeque>(); + /****限制消费令****/ + public static LinkedBlockingDeque> xzxflQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> xzxflErrorQueue = new LinkedBlockingDeque>(); + /****终本案件****/ + public static LinkedBlockingDeque> zbajQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> zbajErrorQueue = new LinkedBlockingDeque>(); + /****终本案件****/ + public static LinkedBlockingDeque> laxxQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> laxxErrorQueue = new LinkedBlockingDeque>(); + /****主要人员****/ + public static LinkedBlockingDeque> zyryQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> zyryErrorQueue = new LinkedBlockingDeque>(); + /****股东信息****/ + public static LinkedBlockingDeque> gdxxQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> gdxxErrorQueue = new LinkedBlockingDeque>(); + /****工商信息****/ + public static LinkedBlockingDeque> gsxxQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> gsxxErrorQueue = new LinkedBlockingDeque>(); + /***任务错误队列**/ + public static LinkedBlockingDeque> taskErrorQueue = new LinkedBlockingDeque>(); + + public static Map getATask( + LinkedBlockingDeque> taskQueue) { + Map task; + try { + task = taskQueue.take(); + return task; + } catch (Exception e) { + return null; + } + } +} diff --git a/src/main/java/com/bfd/analyze/utils/SpringBootKafka.java b/src/main/java/com/bfd/analyze/utils/SpringBootKafka.java new file mode 100644 index 0000000..5771697 --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/SpringBootKafka.java @@ -0,0 +1,48 @@ +package com.bfd.analyze.utils; + +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.SendResult; +import org.springframework.stereotype.Component; +import org.springframework.util.concurrent.ListenableFuture; +import org.springframework.util.concurrent.ListenableFutureCallback; + +/** + * @PROJECT_NAME: companybusinesscrawl + * @DESCRIPTION:SpringBootKafka 工具类 + * @AUTHOR: ying.zhao + * @DATE: 2023/4/6 11:09 + */ +@Slf4j +@Component +public class SpringBootKafka { + @Autowired + private KafkaTemplate kafkaTemplate; + /** + * 自定义topicKafkaTemplate + */ + /** + * public static final String TOPIC = "companyBussTest"; + **/ + public void send(String topic, String message) { + String obj2String = JSONObject.toJSONString(message); + log.info("准备发送消息为：{}", obj2String); + //发送消息 + ListenableFuture> future = kafkaTemplate.send(topic, message); + future.addCallback(new ListenableFutureCallback>() { + @Override + public void onFailure(Throwable throwable) { + //发送失败的处理 + log.info(topic + " - 生产者 发送消息失败：" + throwable.getMessage()); + } + + @Override + public void onSuccess(SendResult stringObjectSendResult) { + //成功的处理 + log.info(topic + " - 生产者 发送消息成功：" + stringObjectSendResult.toString()); + } + }); + } +} diff --git a/src/main/java/com/bfd/analyze/utils/SqlStatementInterceptor.java b/src/main/java/com/bfd/analyze/utils/SqlStatementInterceptor.java new file mode 100644 index 0000000..1774351 --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/SqlStatementInterceptor.java @@ -0,0 +1,226 @@ +package com.bfd.analyze.utils; +//package com.bfd.analyze.utils; +// +//import java.sql.Connection; +//import java.text.DateFormat; +//import java.util.Date; +//import java.util.List; +//import java.util.Locale; +//import java.util.Properties; +//import java.util.regex.Matcher; +// +//import com.alibaba.fastjson.JSONObject; +//import lombok.extern.slf4j.Slf4j; +//import org.apache.commons.collections.CollectionUtils; +//import org.apache.ibatis.executor.statement.StatementHandler; +//import org.apache.ibatis.mapping.BoundSql; +//import org.apache.ibatis.mapping.MappedStatement; +//import org.apache.ibatis.mapping.ParameterMapping; +//import org.apache.ibatis.plugin.Interceptor; +//import org.apache.ibatis.plugin.Intercepts; +//import org.apache.ibatis.plugin.Invocation; +//import org.apache.ibatis.plugin.Plugin; +//import org.apache.ibatis.plugin.Signature; +//import org.apache.ibatis.reflection.DefaultReflectorFactory; +//import org.apache.ibatis.reflection.MetaObject; +//import org.apache.ibatis.reflection.SystemMetaObject; +//import org.apache.ibatis.session.Configuration; +//import org.apache.ibatis.type.TypeHandlerRegistry; +//import org.springframework.beans.factory.annotation.Autowired; +//import org.springframework.kafka.core.KafkaTemplate; +//import org.springframework.kafka.support.SendResult; +//import org.springframework.stereotype.Component; +//import org.springframework.util.concurrent.ListenableFuture; +//import org.springframework.util.concurrent.ListenableFutureCallback; +// +//import javax.annotation.PostConstruct; +///** +// * @author:zhaoying +// * @className:SqlStatementInterceptor +// * @version:1.0 +// * @description: +// * @Date:2023-04-03 14:34:27 +// */ +///** +// * 拦截StatementHandler类中参数类型为Statement的 prepare 方法 +// * 即拦截 Statement prepare(Connection var1, Integer var2) 方法 +// +// */ +// +//@Component +//@Slf4j +//@Intercepts({@Signature(type = StatementHandler.class, method = "prepare", args = {Connection.class, Integer.class})}) +//public class SqlStatementInterceptor implements Interceptor { +// @Autowired +// private KafkaTemplate kafkaTemplate; +// @Override +// public Object intercept(Invocation invocation) throws Throwable { +// +// StatementHandler statementHandler = (StatementHandler) invocation.getTarget(); +// //通过MetaObject优雅访问对象的属性，这里是访问statementHandler的属性;：MetaObject是Mybatis提供的一个用于方便、 +// //优雅访问对象属性的对象，通过它可以简化代码、不需要try/catch各种reflect异常，同时它支持对JavaBean、Collection、Map三种类型对象的操作。 +// MetaObject metaObject = MetaObject +// .forObject(statementHandler, SystemMetaObject.DEFAULT_OBJECT_FACTORY, SystemMetaObject.DEFAULT_OBJECT_WRAPPER_FACTORY, +// new DefaultReflectorFactory()); +// //先拦截到RoutingStatementHandler，里面有个StatementHandler类型的delegate变量，其实现类是BaseStatementHandler，然后就到BaseStatementHandler的成员变量mappedStatement +// MappedStatement mappedStatement = (MappedStatement) metaObject.getValue("delegate.mappedStatement"); +// //id为执行的mapper方法的全路径名，如com.uv.dao.UserMapper.insertUser +// String id = mappedStatement.getId(); +// log.info("id ==> " + id); +// //sql语句类型 select、delete、insert、update +// String sqlCommandType = mappedStatement.getSqlCommandType().toString(); +// log.info("类型 ==> " + sqlCommandType); +// +// BoundSql boundSql = statementHandler.getBoundSql(); +// log.info("boundSql={}",boundSql); +// // 获取节点的配置 +// Configuration configuration = mappedStatement.getConfiguration(); +// // 获取到最终的sql语句 +// String newsql = getSql(configuration, boundSql, id); +// log.info("拦截的sql ==>: " + newsql); +// String insertSql = "INSERT INTO"; +// if(newsql.contains(insertSql)){ +// log.info("是insert语句，发送kafka"); +// newsql = newsql.substring(newsql.indexOf(":")+1); +// send(newsql); +// } +// +// long start = System.currentTimeMillis(); +// Object returnValue = invocation.proceed(); +// long end = System.currentTimeMillis(); +// long time = (end - start); +// log.info("sql耗时 ==>: " + time); +// return returnValue; +// //return null; +// } +// +// @Override +// public Object plugin(Object target) { +// return Plugin.wrap(target, this); +// } +// +// @Override +// public void setProperties(Properties properties) { +// } +// +// +// /** +// * 封装了一下sql语句， +// * 使得结果返回完整xml路径下的sql语句节点id + sql语句 +// * +// * @param configuration +// * @param boundSql +// * @param sqlId +// * @return +// */ +// private String getSql(Configuration configuration, BoundSql boundSql, String sqlId) { +// String sql = showSql(configuration, boundSql); +// StringBuilder str = new StringBuilder(100); +// str.append(sqlId); +// str.append(":"); +// str.append(sql); +// return str.toString(); +// } +// +// /** +// * 如果参数是String，则添加单引号， 如果是日期，则转换为时间格式器并加单引号； +// * 对参数是null和不是null的情况作了处理
+// * +// * @param obj +// * @return +// */ +// private String getParameterValue(Object obj) { +// String value = null; +// if (obj instanceof String) { +// value = "'" + obj.toString() + "'"; +// } else if (obj instanceof Date) { +// DateFormat formatter = DateFormat.getDateTimeInstance(DateFormat.DEFAULT, DateFormat.DEFAULT, Locale.CHINA); +// value = "'" + formatter.format(new Date()) + "'"; +// } else { +// if (obj != null) { +// value = obj.toString(); +// } else { +// value = ""; +// } +// +// } +// return value; +// } +// +// /** +// * 进行？的替换 +// * @param configuration +// * @param boundSql +// * @return +// */ +// public String showSql(Configuration configuration, BoundSql boundSql) { +// // 获取参数 +// Object parameterObject = boundSql.getParameterObject(); +// List parameterMappings = boundSql +// .getParameterMappings(); +// // sql语句中多个空格都用一个空格代替 +// String sql = boundSql.getSql().replaceAll("[\\s]+", " "); +// if (CollectionUtils.isNotEmpty(parameterMappings) && parameterObject != null) { +// // 获取类型处理器注册器，类型处理器的功能是进行java类型和数据库类型的转换　　　　　　　 +// // 如果根据parameterObject.getClass(）可以找到对应的类型，则替换 +// TypeHandlerRegistry typeHandlerRegistry = configuration.getTypeHandlerRegistry(); +// if (typeHandlerRegistry.hasTypeHandler(parameterObject.getClass())) { +// sql = sql.replaceFirst("\\?", Matcher.quoteReplacement(getParameterValue(parameterObject))); +// +// } else { +// //MetaObject主要是封装了originalObject对象， +// // 提供了get和set的方法用于获取和设置originalObject的属性值, +// // 主要支持对JavaBean、Collection、Map三种类型对象的操作 +// MetaObject metaObject = configuration.newMetaObject(parameterObject); +// for (ParameterMapping parameterMapping : parameterMappings) { +// String propertyName = parameterMapping.getProperty(); +// if (metaObject.hasGetter(propertyName)) { +// Object obj = metaObject.getValue(propertyName); +// sql = sql.replaceFirst("\\?", Matcher.quoteReplacement(getParameterValue(obj))); +// } else if (boundSql.hasAdditionalParameter(propertyName)) { +// // 该分支是动态sql +// Object obj = boundSql.getAdditionalParameter(propertyName); +// sql = sql.replaceFirst("\\?", Matcher.quoteReplacement(getParameterValue(obj))); +// +// } else { +// //打印出缺失，提醒该参数缺失并防止错位 +// sql = sql.replaceFirst("\\?", "缺失"); +// } +// } +// } +// } +// return sql; +// } +//// @Autowired +//// private KafkaTemplate kafkaTemplate; +// /** +// * 自定义topicKafkaTemplate +// */ +// public static final String TOPIC = "companyBussTest"; +// +// /** +// * 自定义topic组 +// */ +// public static final String TOPIC_GROUP1 = "topic.group1"; +// +// public void send(String message) { +//// String message = "test"; +// String obj2String = JSONObject.toJSONString(message); +// log.info("准备发送消息为：{}", obj2String); +// //发送消息 +// ListenableFuture> future = kafkaTemplate.send(TOPIC, message); +// future.addCallback(new ListenableFutureCallback>() { +// @Override +// public void onFailure(Throwable throwable) { +// //发送失败的处理 +// log.info(TOPIC + " - 生产者 发送消息失败：" + throwable.getMessage()); +// } +// +// @Override +// public void onSuccess(SendResult stringObjectSendResult) { +// //成功的处理 +// log.info(TOPIC + " - 生产者 发送消息成功：" + stringObjectSendResult.toString()); +// } +// }); +// } +//} diff --git a/src/main/java/com/bfd/analyze/utils/ThrowMessageUtil.java b/src/main/java/com/bfd/analyze/utils/ThrowMessageUtil.java new file mode 100644 index 0000000..e54d1a0 --- /dev/null +++ b/src/main/java/com/bfd/analyze/utils/ThrowMessageUtil.java @@ -0,0 +1,23 @@ +package com.bfd.analyze.utils; + +import java.io.PrintWriter; +import java.io.StringWriter; + +/** + * @author jian.mao + * @date 2023年3月22日 + * @description + */ +public class ThrowMessageUtil { + + /** + * 获取异常信息 + * @param t + * @return + */ + public static String getErrmessage(Throwable t){ + StringWriter stringWriter=new StringWriter(); + t.printStackTrace(new PrintWriter(stringWriter,true)); + return stringWriter.getBuffer().toString(); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..432d485 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,136 @@ +logging: + level: + root: info + path: ./logs +server: + port: 8000 + servlet: + context-path: /analystschedule + tomcat: + uri-encoding: utf-8 + max-connections: 20000 + max-http-form-post-size: 1 + max-threads: 1000 +spring: + application: + name: analystschedule + datasource: + url: jdbc:mysql://172.24.12.126:3306/cda_db?serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8&useSSL=true + username: root + password: baifendian123 + driver-class-name: com.mysql.cj.jdbc.Driver + kafka: + bootstrap-servers: 172.16.12.55:9092,172.16.12.56:9092,172.16.12.57:9092 + producer: + retries: 0 + #当有多个消息需要被发送到同一个分区时，生产者会把它们放在同一个批次里。该参数指定了一个批次可以使用的内存大小，按照字节数计算。 + batch-size: 16384 + # 设置生产者内存缓冲区的大小。 + buffer-memory: 33554432 + # 键的序列化方式 + key-serializer: org.apache.kafka.common.serialization.StringSerializer + # 值的序列化方式 + value-serializer: org.apache.kafka.common.serialization.StringSerializer + # acks=0 ： 生产者在成功写入消息之前不会等待任何来自服务器的响应。 + # acks=1 ： 只要集群的首领节点收到消息，生产者就会收到一个来自服务器成功响应。 + # acks=all ：只有当所有参与复制的节点全部收到消息时，生产者才会收到一个来自服务器的成功响应。 + acks: 1 + consumer: + # 自动提交的时间间隔 在spring boot 2.X 版本中这里采用的是值的类型为Duration 需要符合特定的格式，如1S,1M,2H,5D + auto-commit-interval: 1S + # 该属性指定了消费者在读取一个没有偏移量的分区或者偏移量无效的情况下该作何处理： + # latest（默认值）在偏移量无效的情况下，消费者将从最新的记录开始读取数据（在消费者启动之后生成的记录） + # earliest ：在偏移量无效的情况下，消费者将从起始位置读取分区的记录 + auto-offset-reset: earliest + # 是否自动提交偏移量，默认值是true,为了避免出现重复数据和数据丢失，可以把它设置为false,然后手动提交偏移量 + enable-auto-commit: true + # 键的反序列化方式 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + # 值的反序列化方式 + value-deserializer: org.apache.kafka.common.serialization.StringDeserializer + #消费组 + group-id: test4 + #消费者并发线程数 + concurrency: 4 + #超时时间 + max-poll-interval-ms: 60000 + #listener: + # 在侦听器容器中运行的线程数。 + #concurrency: 5 + #listner负责ack，每调用一次，就立即commit + #ack-mode: manual_immediate + #missing-topics-fatal: false + + redis: + host: 172.24.12.126 + port: 6379 + timeout: 10000 + database: 5 + jedis: + pool: + max-active: 8 # 连接池最大连接数（使用负值表示没有限制） + max-wait: 800 # 连接池最大阻塞等待时间（使用负值表示没有限制） + max-idle: 8 # 连接池中的最大空闲连接 + min-idle: 2 # 连接池中的最小空闲连接 + boot: + admin: + client: + #url: http://10.10.143.85:8000 + url: http://172.16.12.55:8001 + instance: + service-base-url: http://10.10.143.85:8000 + +management: + endpoints: + web: + exposure: + include: "*" + endpoint: + health: + show-details: always + health: + elasticsearch: + enabled: false + + +kafka: + consumer: + #自定义topic + topic: analyze + +mybatis: + type-aliases-package: com.bfd.analyze.model +elasticsearch: + cluster-nodes: 172.16.12.55:9200,172.16.12.56:9200,172.16.12.57:9200 + cluster-name: zh628es-cluster + username: elastic + password: bfd123 + index-name: analyze_00001 + dispatch: + index-name: dispatch_analyze_00001 +zookeeper: + connection-string: 172.16.12.55:2181,172.16.12.56:2181,172.16.12.57:2181 + publish-node: /analyze +task: + dispatch-thread-num: 1 + results-thread-num: 1 + save-thread-num: 1 + queue-size: 10000 + error-task-path: ../data/errorTask.txt + error-result-path: ../data/errorResult.txt + task-queue-path: ../data/taskQueue.txt + result-queue-path: ../data/resultQueue.txt + saveData-queue-path: ../data/saveDataQueue.txt + errorTask-queue-path: ../data/errorTaskQueue.txt + cycle-intv: 180000 #m默认3分钟 + error-time: 3 #应用请求最大重试次数 +#计费配置 +billing: + apps: + 2: 语音识别 + 3: 文字翻译 + 1027: 文档翻译 + 17: 文字识别 + 65: 生成式大模型 + billingUrl: http://172.24.12.126:7076 + aimodelType: 65 diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..511cfd6 --- /dev/null +++ b/src/main/resources/logback-spring.xml @@ -0,0 +1,36 @@ + + + + + + + + + true + + ${logging.level} + + + ${logging.path}/analyzeInfoscheduling.log + + + ${logging.path}/analyzeInfoscheduling.log.%d{yyyy-MM-dd} + 7 + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n + UTF-8 + + + + + + + + diff --git a/src/test/java/com/bfd/AppTest.java b/src/test/java/com/bfd/AppTest.java new file mode 100644 index 0000000..7f816c2 --- /dev/null +++ b/src/test/java/com/bfd/AppTest.java @@ -0,0 +1,20 @@ +package com.bfd; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Unit test for simple App. + */ +public class AppTest +{ + /** + * Rigorous Test :-) + */ + @Test + public void shouldAnswerWithTrue() + { + assertTrue( true ); + } +}