From 56c8545eead4be737ebd32925e511d3c6f8071d9 Mon Sep 17 00:00:00 2001
From: 55007 <55007@maojian>
Date: Tue, 7 Jan 2025 18:18:51 +0800
Subject: [PATCH] =?UTF-8?q?=E6=9F=A5=E7=9C=8B=E7=BB=93=E6=9E=9C=E5=BA=94?=
=?UTF-8?q?=E7=94=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.classpath | 40 +
.gitignore | 4 +
.project | 23 +
README.md | 1 +
pom.xml | 235 ++++++
result_assembly.iml | 23 +
src/main/java/com/bfd/assemble/Application.java | 25 +
.../java/com/bfd/assemble/cache/ConfigCache.java | 35 +
.../assemble/controller/BuildReusltController.java | 42 +
.../java/com/bfd/assemble/entity/Constants.java | 147 ++++
.../java/com/bfd/assemble/handler/MainHandler.java | 112 +++
.../bfd/assemble/process/BuildReustPorcess.java | 100 +++
.../bfd/assemble/service/BuildReusltService.java | 16 +
.../service/impl/BuildReusltServiceImpl.java | 51 ++
src/main/java/com/bfd/assemble/utils/DataUtil.java | 63 ++
src/main/java/com/bfd/assemble/utils/DateUtil.java | 177 ++++
.../java/com/bfd/assemble/utils/DownLoadUtil.java | 906 +++++++++++++++++++++
.../com/bfd/assemble/utils/EncryptionUtil.java | 27 +
src/main/java/com/bfd/assemble/utils/FileUtil.java | 36 +
src/main/java/com/bfd/assemble/utils/JsonUtil.java | 32 +
.../java/com/bfd/assemble/utils/OtherUtils.java | 33 +
.../java/com/bfd/assemble/utils/QueueUtil.java | 18 +
.../com/bfd/assemble/utils/SpringBootKafka.java | 46 ++
.../com/bfd/assemble/utils/ThrowMessageUtil.java | 23 +
src/main/resources/application.yml | 86 ++
src/main/resources/logback-spring.xml | 36 +
src/test/java/com/bfd/AppTest.java | 20 +
27 files changed, 2357 insertions(+)
create mode 100644 .classpath
create mode 100644 .gitignore
create mode 100644 .project
create mode 100644 README.md
create mode 100644 pom.xml
create mode 100644 result_assembly.iml
create mode 100644 src/main/java/com/bfd/assemble/Application.java
create mode 100644 src/main/java/com/bfd/assemble/cache/ConfigCache.java
create mode 100644 src/main/java/com/bfd/assemble/controller/BuildReusltController.java
create mode 100644 src/main/java/com/bfd/assemble/entity/Constants.java
create mode 100644 src/main/java/com/bfd/assemble/handler/MainHandler.java
create mode 100644 src/main/java/com/bfd/assemble/process/BuildReustPorcess.java
create mode 100644 src/main/java/com/bfd/assemble/service/BuildReusltService.java
create mode 100644 src/main/java/com/bfd/assemble/service/impl/BuildReusltServiceImpl.java
create mode 100644 src/main/java/com/bfd/assemble/utils/DataUtil.java
create mode 100644 src/main/java/com/bfd/assemble/utils/DateUtil.java
create mode 100644 src/main/java/com/bfd/assemble/utils/DownLoadUtil.java
create mode 100644 src/main/java/com/bfd/assemble/utils/EncryptionUtil.java
create mode 100644 src/main/java/com/bfd/assemble/utils/FileUtil.java
create mode 100644 src/main/java/com/bfd/assemble/utils/JsonUtil.java
create mode 100644 src/main/java/com/bfd/assemble/utils/OtherUtils.java
create mode 100644 src/main/java/com/bfd/assemble/utils/QueueUtil.java
create mode 100644 src/main/java/com/bfd/assemble/utils/SpringBootKafka.java
create mode 100644 src/main/java/com/bfd/assemble/utils/ThrowMessageUtil.java
create mode 100644 src/main/resources/application.yml
create mode 100644 src/main/resources/logback-spring.xml
create mode 100644 src/test/java/com/bfd/AppTest.java
diff --git a/.classpath b/.classpath
new file mode 100644
index 0000000..1a0a8d2
--- /dev/null
+++ b/.classpath
@@ -0,0 +1,40 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..97be0c6
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+/target/
+/.idea/
+/logs/
+/.settings/
diff --git a/.project b/.project
new file mode 100644
index 0000000..7a00cc3
--- /dev/null
+++ b/.project
@@ -0,0 +1,23 @@
+
+
+ result_assembly
+
+
+
+
+
+ org.eclipse.jdt.core.javabuilder
+
+
+
+
+ org.eclipse.m2e.core.maven2Builder
+
+
+
+
+
+ org.eclipse.jdt.core.javanature
+ org.eclipse.m2e.core.maven2Nature
+
+
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..6fcde55
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+结果组装整合应用
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..869bea4
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,235 @@
+
+
+
+ 4.0.0
+
+
+ org.springframework.boot
+ spring-boot-starter-parent
+ 2.2.4.RELEASE
+
+
+ com.bfd
+ result_assembly
+ 0.0.1-SNAPSHOT
+
+ result_assembly
+
+ http://www.example.com
+
+
+ UTF-8
+ 1.8
+ 1.8
+
+
+
+
+ junit
+ junit
+ 4.11
+ test
+
+
+ org.springframework.boot
+ spring-boot-starter-web
+
+
+ com.google.code.gson
+ gson
+ 2.8.8
+
+
+ org.springframework.boot
+ spring-boot-test
+
+
+
+ org.springframework
+ spring-test
+ 5.0.10.RELEASE
+ test
+
+
+
+ de.codecentric
+ spring-boot-admin-starter-client
+ 2.2.4
+
+
+ commons-io
+ commons-io
+ 1.4
+
+
+ com.alibaba
+ fastjson
+ 2.0.17
+
+
+
+ com.mchange
+ c3p0
+ 0.9.5.5
+
+
+ mysql
+ mysql-connector-java
+ 8.0.29
+
+
+
+ com.squareup.okhttp3
+ okhttp
+ 4.9.3
+
+
+ org.apache.httpcomponents
+ httpclient
+ 4.5.3
+
+
+ commons-lang
+ commons-lang
+ 2.6
+
+
+
+ org.jetbrains.kotlin
+ kotlin-reflect
+ 1.6.21
+ runtime
+
+
+
+ org.jsoup
+ jsoup
+ 1.8.1
+
+
+
+ org.projectlombok
+ lombok
+
+
+ org.springframework.kafka
+ spring-kafka
+
+
+
+
+ cn.hutool
+ hutool-all
+ 5.8.5
+
+
+ junit
+ junit
+
+
+
+ p6spy
+ p6spy
+ 3.9.0
+
+
+
+ commons-collections
+ commons-collections
+ 3.2.2
+
+
+
+
+
+
+
+
+
+ maven-clean-plugin
+ 3.1.0
+
+
+
+ maven-resources-plugin
+ 3.0.2
+
+
+ maven-compiler-plugin
+ 3.8.0
+
+
+ maven-surefire-plugin
+ 2.22.1
+
+
+ maven-jar-plugin
+ 3.0.2
+
+
+ maven-install-plugin
+ 2.5.2
+
+
+ maven-deploy-plugin
+ 2.8.2
+
+
+
+ maven-site-plugin
+ 3.7.1
+
+
+ maven-project-info-reports-plugin
+ 3.0.0
+
+
+
+
+ org.springframework.boot
+ spring-boot-maven-plugin
+
+ com.bfd.assemble.Application
+ ZIP
+
+
+ ${project.groupId}
+ ${project.artifactId}
+
+
+
+
+
+
+ repackage
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+ 3.1.1
+
+
+ copy
+ package
+
+ copy-dependencies
+
+
+ jar
+ jar
+ runtime
+ ${project.build.directory}/libs
+
+
+
+
+
+
+
+
diff --git a/result_assembly.iml b/result_assembly.iml
new file mode 100644
index 0000000..e8cf8de
--- /dev/null
+++ b/result_assembly.iml
@@ -0,0 +1,23 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/src/main/java/com/bfd/assemble/Application.java b/src/main/java/com/bfd/assemble/Application.java
new file mode 100644
index 0000000..b6b936b
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/Application.java
@@ -0,0 +1,25 @@
+package com.bfd.assemble;
+
+
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.kafka.annotation.EnableKafka;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+/**
+ * 主入口
+ *
+ * @author jian.mao
+ * @date 2023年7月4日
+ * @description
+ */
+@SpringBootApplication
+@EnableScheduling
+@EnableKafka
+public class Application {
+
+ public static void main(String[] args) {
+ SpringApplication.run(Application.class, args);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/assemble/cache/ConfigCache.java b/src/main/java/com/bfd/assemble/cache/ConfigCache.java
new file mode 100644
index 0000000..f3dedca
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/cache/ConfigCache.java
@@ -0,0 +1,35 @@
+package com.bfd.assemble.cache;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingDeque;
+
+/**
+ * @author jian.mao
+ * @date 2022年11月11日
+ * @description 静态变量类
+ */
+@Slf4j
+public class ConfigCache {
+
+ /**启动条件**/
+ public static boolean isStart = true;
+ /*****任务队列*****/
+ public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>();
+
+
+ /**
+ * 队列录入任务
+ * @param queue
+ * @param task
+ */
+ public static void putQueue(LinkedBlockingDeque> queue,Map task){
+ //next app 写入队列准备调出
+ try {
+ queue.put(task);
+ } catch (InterruptedException e) {
+ log.error("队列写入data失败---");
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/controller/BuildReusltController.java b/src/main/java/com/bfd/assemble/controller/BuildReusltController.java
new file mode 100644
index 0000000..665f2fc
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/controller/BuildReusltController.java
@@ -0,0 +1,42 @@
+package com.bfd.assemble.controller;
+
+import com.bfd.assemble.service.BuildReusltService;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import javax.annotation.Resource;
+
+/**
+ * @PROJECT_NAME: result_assembly
+ * @DESCRIPTION:
+ * @AUTHOR: jian.mao
+ * @DATE: 2023/11/1 16:12
+ */
+
+@Controller
+@RequestMapping("/build")
+@Slf4j
+public class BuildReusltController {
+
+ @Resource
+ private BuildReusltService buildReusltService;
+ @PostMapping("/result")
+ @ResponseBody
+ public String buildResult(@RequestBody String dataJson){
+ String result = buildReusltService.buildReulst(dataJson);
+ return result;
+ }
+ @RequestMapping(value = "/hello", method = RequestMethod.GET)
+ @ResponseBody
+ public String hello(String param, String token) {
+ return "123";
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/entity/Constants.java b/src/main/java/com/bfd/assemble/entity/Constants.java
new file mode 100644
index 0000000..9901a11
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/entity/Constants.java
@@ -0,0 +1,147 @@
+package com.bfd.assemble.entity;
+
+
+/**
+ * 常量实体类
+ * @author jian.mao
+ * @date 2022年11月15日
+ * @description
+ */
+public class Constants {
+
+ /*************************蓝图常量key名称*********************************/
+ public final static String SCHEDULING = "scheduling";
+ public final static String TYPE = "type";
+ public final static String INTERVAL = "interval";
+ public final static String CREATED = "created";
+ public final static String LAST_EDIT = "last_edit";
+ public final static String BLUEPRINT_ID = "blueprint_id";
+ public final static String BLUEPRINTID = "blueprintId";
+ public final static String BLUEPRINT_NAME = "name";
+ public final static String SCENARIO = "scenario";
+ public final static String AUTOCOMMITTRIGGERLAST = "autoCommitTriggerLast";
+ public final static String FRESHVARIABLES = "freshVariables";
+ public final static String AUTOCOMMIT = "autoCommit";
+ public final static String MAXERRORS = "maxErrors";
+ public final static String DATALOSS = "dataloss";
+ public final static String POSITION = "position";
+ public final static String SCENES_ID = "scenes_id";
+ public final static String SCENESID = "scenesId";
+ public final static String MULTI_BRANCH = "multi_branch";
+
+ public final static String SINGLE = "single";
+ /**已重试次数**/
+ public final static String ERROR_TIME = "error_time";
+ public final static String PREVIOUS_RESULT = "previous_result";
+
+ /****数据id*****/
+ public final static String BUSINESSKEY = "businessKey";
+
+
+ /*************************metadata常量key名称*********************************/
+ public final static String LABEL_COL = "label_col";
+ public final static String LABEL = "label";
+ public final static String USER = "user";
+ public final static String ADMIN = "admin";
+ public final static String ADDRESS = "address";
+ public final static String DATASOURCE = "datasource";
+ public final static String INDEX = "index";
+
+ /*************************app常量key名称*********************************/
+ public final static String APPS = "apps";
+ public final static String TRANSFER_ID = "transfer_id";
+ public final static String MODULE = "module";
+ public final static String VERSION = "version";
+ public final static String METADATA = "metadata";
+ public final static String APP_NAME = "name";
+ public final static String DESCRIBE = "describe";
+ public final static String NEXT_APP_ID = "next_app_id";
+ public final static String EDGE_ID = "edge_id";
+ public final static String START_ID = "start_id";
+ public final static String END_ID = "end_id";
+
+ public final static String WAIT_CONDITION = "wait_condition";
+ public final static String START_TAG = "start_tag";
+
+ /*************************module类型*********************************/
+ public final static String FILE = "file";
+ public final static String OCR = "OCR";
+ public final static String FILTER = "Filter";
+ public final static String CHATGPT = "ChatGPT";
+ public final static String MYSQL = "mysql";
+
+ /*************************other类型*********************************/
+ public final static String UNDERLINE = "_";
+ public final static String RESULT_TOPIC = null;
+ public static final String EMPTY = "";
+ public static final String HTTP = "http";
+ public static final String REQUEST_ERROR_MESSAGE = "Download failed error is";
+ public static final String REQUEST_RESULT = "result";
+ public static final String REQUEST_RESULT_RESULTS = "results";
+ public static final String MAP_TYPE = "Map";
+ public static final String LIST_TYPE = "List";
+ public static final String STRING_TYPE = "String";
+ public static final String DOCUMENT_TYPE = "doc";
+ public static final String FILTER_ZH = "过滤器";
+
+ public static final String JSON_SELE_SYMBOL = "$.";
+ public static final String LEFT_BRACKETS = "[";
+ public static final String RIGTH_BRACKETS = "]";
+ public static final String TASKTYPE = "taskType";
+ public static final Integer USER_TYPE = 1;
+ public static final Integer KEYWORD_TYPE = 0;
+ public static final Integer DETAIL_TYPE = 2;
+ public static final String CID = "cid";
+ public static final String SITETYPE = "siteType";
+ public static final Integer DEFULT_SUBJECTID = 304864;
+ public static final Integer DEFULT_CRAWLCYCLICITYTIME = 1440;
+ public static final String CRAWLENDTIME = "crawlEndTime";
+ public static final String CRAWLSTARTTIME = "crawlStartTime";
+ public static final String CRAWLPAGETYPES = "crawlPageTypes";
+ public static final String APPID = "113ic";
+ public static final String APP_ID = "appId";
+ public final static String ID = "id";
+ public static final Integer DEFULT_CRAWLPERIODHOUR = 24;
+ public static final String CREATEUSERID = "662015832180933762";
+ public static final String CRAWL_ADD_URL = "https://caiji.percent.cn/api/crawl/remote/task/save";
+ public static final String CRAWLKEYWORD = "crawlKeyword";
+ public static final String ATTACHTAG = "attachTag";
+ public static final String ATTACHTAG_VALUE = "analyze";
+ public static final String KEYWORD = "keyword";
+ public static final String SITEID = "siteId";
+ public static final String RESULTS = "results";
+ public static final String RESULT = "result";
+ public static final String CRAWLDATAFLAG = "crawlDataFlag";
+ public static final String CRAWLDATAFLAG_PREFIX = "\"crawlDataFlag\":\"keyword:";
+ public static final String TID = "tid";
+ public static final Long TIME_OUT = 1800000L;
+ public static final String ATTR = "attr";
+ public static final String HASVIDEO = "hasVideo";
+ public static final String CRAWL_END_MARK = "crawl_end_mark";
+ public static final String CRAWL_END_MESSAGE = "crawl_end_message";
+ public static final String CRAWL_END_MESSAGE_VALUE = "数据采集完成";
+ public static final String SUBJECTID = "subjectId";
+ public static final String TASKID = "taskId";
+ public static final int SUCCESS_CODE = 200;
+ public static final String WEB_URL_SUFFIX = "/api/aogeo/api/cda/caiji/status";
+ public static final String STATUS = "status";
+ /************************redis*************************************/
+ public static final String LOCK_KEY = "myLock";
+ public static final long LOCK_EXPIRE_TIME = 300000;
+
+ /************************应用参数*************************************/
+ public static final String CODE = "code";
+ public static final String MESSAGE = "message";
+ public static final String INPUT = "input";
+ public static final String OUTPUT = "output";
+ public static final String FORM = "form";
+ public static final String FIELD = "field";
+ public static final String DATA = "data";
+ public static final String COLON_EN = ":";
+ public static final String FLAGSEARCH = "flagSearch";
+ public static final String KEY = "key";
+ public static final String VALUE = "value";
+ public static final String SEARCHFIELDS = "searchFields";
+ public static final String ISLAST = "isLast";
+ public static final String TRACE = "trace";
+}
diff --git a/src/main/java/com/bfd/assemble/handler/MainHandler.java b/src/main/java/com/bfd/assemble/handler/MainHandler.java
new file mode 100644
index 0000000..0c1a669
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/handler/MainHandler.java
@@ -0,0 +1,112 @@
+package com.bfd.assemble.handler;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import javax.annotation.Resource;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.io.FileUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
+import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.assemble.cache.ConfigCache;
+import com.bfd.assemble.process.BuildReustPorcess;
+import com.bfd.assemble.utils.FileUtil;
+
+
+/**
+ * 启动处理入口
+ * @author jian.mao
+ * @date 2023年11月3日
+ * @description
+ */
+
+@Component
+@Order(value = 1)
+@Slf4j
+public class MainHandler implements ApplicationRunner {
+
+ @Value("${task.task-queue-path}")
+ private String taskPath;
+ @Resource
+ private BuildReustPorcess buildReustPorcess;
+ @Override
+ public void run(ApplicationArguments args) throws Exception {
+
+ new Thread(buildReustPorcess).start();;
+ log.info("开启问答处理线程-----");
+ //停止处理
+ waitDown();
+ //启动加载缓存任务
+ readTask(taskPath,ConfigCache.taskQueue);
+ }
+
+
+
+
+
+ @SuppressWarnings("unchecked")
+ public static void readTask(String path,LinkedBlockingDeque> queue){
+ File file = new File(path);
+ if(file.exists()){
+ List tasks = null;
+ try {
+ tasks = FileUtils.readLines(file,"UTF-8");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ for (String taskStr : tasks) {
+ Map task = JSONObject.parseObject(taskStr);
+ try {
+ queue.put(task);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ file.delete();
+ }
+ }
+ /**
+ * 结束触发钩子
+ */
+ public void waitDown() {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ // 停止线程
+ ConfigCache.isStart = false;
+ log.info("stop-------");
+ writeTsskToFile();
+ }
+ });
+ }
+
+
+ /**
+ * 任务持久化到硬盘
+ */
+ public void writeTsskToFile(){
+ while(true){
+ if(ConfigCache.taskQueue.size() > 0 ){
+ try {
+ Map task = ConfigCache.taskQueue.take();
+ FileUtil.writeFile(taskPath, JSONObject.toJSONString(task));
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }else{
+ log.info("taskQueue write is file end");
+ break;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/process/BuildReustPorcess.java b/src/main/java/com/bfd/assemble/process/BuildReustPorcess.java
new file mode 100644
index 0000000..48353d1
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/process/BuildReustPorcess.java
@@ -0,0 +1,100 @@
+package com.bfd.assemble.process;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.assemble.cache.ConfigCache;
+import com.bfd.assemble.entity.Constants;
+import com.bfd.assemble.utils.DataUtil;
+import com.bfd.assemble.utils.SpringBootKafka;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * @PROJECT_NAME: result_assembly
+ * @DESCRIPTION:
+ * @AUTHOR: jian.mao
+ * @DATE: 2023/11/1 17:06
+ */
+@Component
+@Slf4j
+public class BuildReustPorcess implements Runnable{
+ @Autowired
+ private SpringBootKafka springBootKafka;
+ @Value("${customize-kafka.producer.topic}")
+ private String topic;
+ @Override
+ public void run() {
+ while (ConfigCache.isStart){
+ Map task = null;
+ try {
+ task = ConfigCache.taskQueue.take();
+ //输入字段
+ Map input = (Map) task.get(Constants.INPUT);
+ //输出字段
+ Map output = (Map) task.get(Constants.OUTPUT);
+ //data
+ Map data = (Map) task.get(Constants.DATA);
+ //结果集
+ Map result = new HashMap(16);
+ Map results = new HashMap(16);
+ //字段遍历
+ List> form = (List>) input.get(Constants.FORM);
+ //检索字段存储对象
+ List> searchFields = new ArrayList>();
+ for (Map map : form) {
+ String field = (String) map.get(Constants.FIELD);
+ String value = (String) map.get(Constants.VALUE);
+ Object parsedvalue = DataUtil.getValue(value,data);
+ //字段为空不处理
+ if(parsedvalue == null){
+ continue;
+ }
+ //支持检索
+ if ((int)map.get(Constants.FLAGSEARCH) > 0){
+ Map searchField = new HashMap(16);
+ searchField.put(Constants.KEY, field);
+ searchField.put(Constants.VALUE, parsedvalue.toString());
+ searchFields.add(searchField);
+ }
+ if(output.containsKey(field)){
+ results.put(field, parsedvalue.toString());
+ }
+ }
+ result.put(Constants.SEARCHFIELDS, searchFields);
+ results.put(Constants.ISLAST, 1);
+ result.put(Constants.RESULTS, JSONObject.toJSONString(results));
+ result.put(Constants.MESSAGE, "成功");
+ result.put(Constants.STATUS, 1);
+ task.put(Constants.RESULT, result);
+ //发送kafka
+ springBootKafka.send(topic,JSONObject.toJSONString(task));
+ log.info("数据流转至下游-------");
+ } catch (Exception e) {
+ // TODO: handle exception
+ log.error("结果组装异常，",e);
+ //结果集
+ Map result = new HashMap(16);
+ Map results = new HashMap(16);
+ //遍历入库返回结果，拼接响应内容
+ results.put(Constants.ISLAST, 1);
+ result.put(Constants.RESULTS, JSONObject.toJSONString(results));
+ result.put(Constants.MESSAGE, "异常");
+ result.put(Constants.STATUS, 2);
+ task.put(Constants.RESULT, result);
+ //发送kafka
+ springBootKafka.send(topic, JSONObject.toJSONString(task));
+ log.info("数据流转至下游-------");
+ }
+
+ }
+ }
+
+}
diff --git a/src/main/java/com/bfd/assemble/service/BuildReusltService.java b/src/main/java/com/bfd/assemble/service/BuildReusltService.java
new file mode 100644
index 0000000..6cba9be
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/service/BuildReusltService.java
@@ -0,0 +1,16 @@
+package com.bfd.assemble.service;
+
+/**
+ * @PROJECT_NAME: result_assembly
+ * @DESCRIPTION:
+ * @AUTHOR: jian.mao
+ * @DATE: 2023/11/1 16:27
+ */
+public interface BuildReusltService {
+ /**
+ * 组装结果
+ * @param dataJson 请求参数
+ * @return
+ */
+ public String buildReulst(String dataJson);
+}
diff --git a/src/main/java/com/bfd/assemble/service/impl/BuildReusltServiceImpl.java b/src/main/java/com/bfd/assemble/service/impl/BuildReusltServiceImpl.java
new file mode 100644
index 0000000..7893b67
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/service/impl/BuildReusltServiceImpl.java
@@ -0,0 +1,51 @@
+package com.bfd.assemble.service.impl;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.assemble.cache.ConfigCache;
+import com.bfd.assemble.entity.Constants;
+import com.bfd.assemble.service.BuildReusltService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @PROJECT_NAME: result_assembly
+ * @DESCRIPTION:
+ * @AUTHOR: jian.mao
+ * @DATE: 2023/11/1 16:37
+ */
+@Service
+@Slf4j
+public class BuildReusltServiceImpl implements BuildReusltService {
+
+ @Override
+ public String buildReulst(String dataJson) {
+ Map response = new HashMap<>(16);
+ int code = 200;
+ String message = "success";
+ Map task = null;
+ try {
+ task = JSONObject.parseObject(dataJson);
+ } catch (Exception e) {
+ log.error("参数结构不合法，",e);
+ code = 100010;
+ message = "参数不合法";
+ }
+ //写入队列
+ try {
+ if(task.containsKey(Constants.TRACE) && (boolean)task.get(Constants.TRACE)){
+ ConfigCache.taskQueue.putFirst(task);
+ }else{
+ ConfigCache.taskQueue.put(task);
+ }
+ } catch (InterruptedException e) {
+ log.error("任务写入等待队列异常，",e);
+ code = 100011;
+ message = "任务写入等待队列失败";
+ }
+ response.put(Constants.CODE,code);
+ response.put(Constants.MESSAGE,message);
+ return JSONObject.toJSONString(response);
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/utils/DataUtil.java b/src/main/java/com/bfd/assemble/utils/DataUtil.java
new file mode 100644
index 0000000..ad6f5e3
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/utils/DataUtil.java
@@ -0,0 +1,63 @@
+package com.bfd.assemble.utils;
+
+import java.util.Map;
+
+import lombok.extern.slf4j.Slf4j;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.JSONPath;
+import com.bfd.assemble.entity.Constants;
+
+/**
+ * @author:jinming
+ * @className:DataUtil
+ * @version:1.0
+ * @description: 获取dataValue的值
+ * @Date:2023/11/1 9:54
+ */
+@Slf4j
+public class DataUtil {
+ /**
+ *
+ * @param key 传入的key
+ * @param dataMap 数据map
+ * @return 根据传入的参数进行判断解析，返回正确的dataValue
+ */
+ public static Object getValue(String key, Map dataMap) {
+ try {
+ //公式为空直接就返回
+ if(key.equals(Constants.EMPTY)){
+ return Constants.EMPTY;
+ }
+ Object dataValue;
+ String isJson = "#json#";
+ if (key.contains(isJson)) {
+ //进行第一次拆分，获取#json#前面的部分
+ String[] keySplit = key.split(isJson);
+ String firstDataKey = keySplit[0];
+ String[] firstDataKeySplit = firstDataKey.split(":");
+ //取出前半部分对应的JSON数据并转换为JSONObject
+ String dataJson = (String) dataMap.get(firstDataKeySplit[0]);
+ JSONObject dataJsonObject = JSON.parseObject(dataJson);
+ //根据key的后半部分取出对应JSONObject中的值
+ String firstDataKeyJson = (String) JSONPath.eval(dataJsonObject, firstDataKeySplit[1]);
+ String secDataKey = keySplit[1];
+ JSONObject firstDataJsonObject = JSON.parseObject(firstDataKeyJson);
+ dataValue = JSONPath.eval(firstDataJsonObject, secDataKey);
+ return dataValue;
+ }
+ String[] keySplit = key.split(":");
+ String jsonPath = keySplit[1];
+ String dataJson = (String) dataMap.get(keySplit[0]);
+ JSONObject dataJsonObject = JSON.parseObject(dataJson);
+ dataValue = JSONPath.eval(dataJsonObject, jsonPath);
+ return dataValue;
+ } catch (Exception e) {
+ // TODO: handle exception
+ log.error("jsonpath公式取值异常，",e);
+ return null;
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/assemble/utils/DateUtil.java b/src/main/java/com/bfd/assemble/utils/DateUtil.java
new file mode 100644
index 0000000..969c6d7
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/utils/DateUtil.java
@@ -0,0 +1,177 @@
+package com.bfd.assemble.utils;
+
+
+import java.math.BigInteger;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+
+import lombok.extern.slf4j.Slf4j;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+
+/**
+ * 日期工具类
+ *
+ * @author jian.mao
+ * @date 2022年11月15日
+ * @description
+ */
+@Slf4j
+public class DateUtil {
+
+ /**
+ * @return
+ */
+ public static String getTimeStrForNow() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH");
+ return sdf.format(new Date());
+ }
+
+
+ public static String getTimeStrForDay(long time) {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+
+ return sdf.format(new Date(time * 1000));
+ }
+
+ public static String getTimeStrForDay() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+
+ return sdf.format(new Date());
+ }
+
+
+ public static String getDateTime() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String time = sdf.format(new Date());
+ return time;
+ }
+
+ public static String getDateTime(Long timestap) {
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String time = sdf.format(new Date(timestap));
+ return time;
+ }
+
+ public static String getDate(Long timestap) {
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+ String time = sdf.format(new Date(timestap));
+ return time;
+ }
+
+ public static String getDateTimeForMonth() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM");
+ String time = sdf.format(new Date());
+ return time;
+ }
+
+ /**
+ * 休眠
+ *
+ * @param millis 毫秒
+ */
+ public static void sleep(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * 1. @Description:时间戳转时间
+ * 2. @Author: ying.zhao
+ * 3. @Date: 2023/3/28
+ */
+
+ public static String timestampToDate(String time) {
+ int thirteen = 13;
+ int ten = 10;
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+// if (time.length() == thirteen) {
+ if (time.length() > ten) {
+ return sdf.format(new Date(Long.parseLong(time)));
+ } else {
+ return sdf.format(new Date(Integer.parseInt(time) * 1000L));
+ }
+ }
+
+ public static String parseCreated(String jsonTime){
+ String formattedDateTime = getDateTime();
+ try {
+ // 使用fastjson解析JSON数据
+ JSONObject jsonObject = JSON.parseObject(jsonTime);
+ // 获取日期和时间的值
+ JSONObject dateObject = jsonObject.getJSONObject("date");
+ int day = dateObject.getIntValue("day");
+ int month = dateObject.getIntValue("month");
+ int year = dateObject.getIntValue("year");
+
+ JSONObject timeObject = jsonObject.getJSONObject("time");
+ int hour = timeObject.getIntValue("hour");
+ int minute = timeObject.getIntValue("minute");
+ int second = timeObject.getIntValue("second");
+
+ // 创建LocalDateTime对象
+ LocalDateTime dateTime = LocalDateTime.of(year, month, day, hour, minute, second);
+
+ // 定义日期时间格式化器
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+ // 格式化日期时间
+ formattedDateTime = dateTime.format(formatter);
+ } catch (Exception e) {
+ log.info("日期转换失败：{}",e);
+ }
+ return formattedDateTime;
+ }
+
+ /**
+ * 字符串转换日期
+ * @param format
+ * @param date
+ * @return
+ */
+ public static Date strToDate(String format,String date){
+ SimpleDateFormat sdf = new SimpleDateFormat(format);
+ if (date == null || date.equals("")){
+ return new Date();
+ }else{
+ Date ru = null;
+ try {
+ ru = sdf.parse(date);
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+ return ru;
+ }
+ }
+ /**
+ * 日期格式话
+ * @param format 日期格式
+ * @param dater 要转换的日期,默认当前时间
+ * @return
+ */
+ public static String FormatDate(String format,Date date){
+ String fromatDate = null;
+ SimpleDateFormat sdf = new SimpleDateFormat(format);
+ if (date == null){
+ fromatDate = sdf.format(new Date());
+ }else{
+ fromatDate = sdf.format(date);
+ }
+ return fromatDate;
+ }
+ public static void main(String[] args) {
+ String time = timestampToDate("955814400000");
+ System.out.println(time);
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/utils/DownLoadUtil.java b/src/main/java/com/bfd/assemble/utils/DownLoadUtil.java
new file mode 100644
index 0000000..8dbc07d
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/utils/DownLoadUtil.java
@@ -0,0 +1,906 @@
+package com.bfd.assemble.utils;
+
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.StatusLine;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.AuthCache;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.config.SocketConfig;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.LaxRedirectStrategy;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.bfd.assemble.entity.Constants;
+
+
+
+
+
+
+/**
+ * 下载工具类
+ * @author jian.mao
+ * @date 2023年9月19日
+ * @description
+ */
+public class DownLoadUtil {
+
+ private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36";
+ private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class);
+ /** 代理服务器(产品官网 www.16yun.cn) **/
+ final static String PROXYHOST = "u270.40.tp.16yun.cn";
+ final static Integer PROXYPORT = 6448;
+ /** 代理验证信息 **/
+ final static String PROXYUSER = "16HFBVJC";
+ final static String PROXYPASS = "897944";
+
+ private static PoolingHttpClientConnectionManager cm = null;
+ private static HttpRequestRetryHandler httpRequestRetryHandler = null;
+ private static HttpHost proxy = null;
+
+ private static CredentialsProvider credsProvider = null;
+ private static RequestConfig reqConfig = null;
+
+ static {
+ ConnectionSocketFactory plainsf = PlainConnectionSocketFactory
+ .getSocketFactory();
+ LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory
+ .getSocketFactory();
+
+ Registry registry = RegistryBuilder.create().register("http", plainsf)
+ .register("https", sslsf).build();
+
+ cm = new PoolingHttpClientConnectionManager(registry);
+ cm.setMaxTotal(20);
+ cm.setDefaultMaxPerRoute(5);
+
+ proxy = new HttpHost(PROXYHOST, PROXYPORT, "https");
+
+ credsProvider = new BasicCredentialsProvider();
+ credsProvider.setCredentials(AuthScope.ANY,
+ new UsernamePasswordCredentials(PROXYUSER, PROXYPASS));
+
+ reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000)
+ .setConnectTimeout(5000).setSocketTimeout(5000)
+ .setExpectContinueEnabled(false)
+ .setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build();
+ }
+
+ /**
+ * 模拟客户端get请求
+ *
+ * @param url
+ * 模拟请求得url
+ * @param headers
+ * 头部信息，没有可以不传
+ * @return
+ */
+ @SafeVarargs
+ public static String proxyDoGet(String url, Map... headers) {
+ // 设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom()
+ .setConnectTimeout(timeout * 1000)
+ .setConnectionRequestTimeout(timeout * 1000)
+ .setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(timeout * 1000)
+ .setTcpNoDelay(true).build();
+ AuthCache authCache = new BasicAuthCache();
+ authCache.put(proxy, new BasicScheme());
+ HttpClientContext localContext = HttpClientContext.create();
+ localContext.setAuthCache(authCache);
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ CloseableHttpClient httpClient = httpBuilder
+ .setDefaultSocketConfig(socketConfig)
+ .setDefaultRequestConfig(config)
+ .setDefaultCredentialsProvider(credsProvider).build();
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setConfig(reqConfig);
+ if (headers != null && headers.length > 0) {
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key, tempHeaders.get(key).toString());
+ }
+ } else {
+ httpGet.setHeader("Accept",
+ "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
+ }
+ CloseableHttpResponse response = null;
+ String html = "";
+ int notFundCode = 404;
+ int successCode = 200;
+ try {
+ response = httpClient.execute(httpGet, localContext);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if (statusLine.getStatusCode() == successCode) {
+ if (responseEntity != null) {
+ html = EntityUtils.toString(responseEntity, "utf-8");
+ System.out.println("响应内容长度为:"
+ + responseEntity.getContentLength());
+ // 下载结果为空不正常
+ if (html.equals(Constants.EMPTY)) {
+ html = "Download failed error is:reslut is null";
+ }
+ }
+ } else if (statusLine.getStatusCode() == notFundCode) {
+ html = "页面404，正常结束请求即可
";
+ } else {
+ throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:reslut is null";
+ }finally{
+ try {
+ response.close();
+ httpClient.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ return html;
+
+ }
+
+
+ public static String httpsslProxyGet(String url, Map... headers) throws Exception {
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connManager.setMaxTotal(50);
+ connManager.setDefaultMaxPerRoute(10);
+ HttpClients.custom().setConnectionManager(connManager);
+ // 设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom()
+ .setConnectTimeout(timeout * 1000)
+ .setConnectionRequestTimeout(timeout * 1000)
+ .setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(timeout * 1000)
+ .setTcpNoDelay(true).build();
+ AuthCache authCache = new BasicAuthCache();
+ authCache.put(proxy, new BasicScheme());
+ HttpClientContext localContext = HttpClientContext.create();
+ localContext.setAuthCache(authCache);
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ CloseableHttpClient httpClient = httpBuilder
+ .setConnectionManager(connManager)
+ .setDefaultSocketConfig(socketConfig)
+ .setDefaultRequestConfig(config)
+ .setDefaultCredentialsProvider(credsProvider).build();
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setConfig(reqConfig);
+ if (headers != null && headers.length > 0) {
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key, tempHeaders.get(key).toString());
+ }
+ } else {
+ httpGet.setHeader("Accept",
+ "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
+ }
+ CloseableHttpResponse response = null;
+ String html = "";
+ int notFundCode = 404;
+ int successCode = 200;
+ try {
+ response = httpClient.execute(httpGet, localContext);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if (statusLine.getStatusCode() == successCode) {
+ if (responseEntity != null) {
+ html = EntityUtils.toString(responseEntity, "utf-8");
+ System.out.println("响应内容长度为:"
+ + responseEntity.getContentLength());
+ // 下载结果为空不正常
+ if (html.equals(Constants.EMPTY)) {
+ html = "Download failed error is:reslut is null";
+ }
+ }
+ } else if (statusLine.getStatusCode() == notFundCode) {
+ html = "页面404，正常结束请求即可
";
+ } else {
+ throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:reslut is null";
+ }finally{
+ try {
+ response.close();
+ httpClient.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ return html;
+
+ }
+
+
+ /**
+ * json参数方式POST提交
+ * @param url
+ * @param params
+ * @return
+ */
+ public static String doPost(String url, String params){
+ String strResult = "";
+ //设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(timeout * 1000)
+ .setTcpNoDelay(true).build();
+// AuthCache authCache = new BasicAuthCache();
+// authCache.put(proxy, new BasicScheme());
+// HttpClientContext localContext = HttpClientContext.create();
+// localContext.setAuthCache(authCache);
+ // 1. 获取默认的client实例
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ httpBuilder.setUserAgent(ua);
+ HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
+// HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
+// .setDefaultCredentialsProvider(credsProvider).build();
+ // 2. 创建httppost实例
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
+ HttpResponse resp = null;
+ try {
+ httpPost.setEntity(new StringEntity(params,"utf-8"));
+ resp = client.execute(httpPost);
+// resp = client.execute(httpPost,localContext);
+ StatusLine statusLine = resp.getStatusLine();
+ System.out.println("响应状态为:" + resp.getStatusLine());
+ int notFundCode = 404;
+ int successCode = 200;
+ if(statusLine.getStatusCode() == successCode){
+ // 7. 获取响应entity
+ HttpEntity respEntity = resp.getEntity();
+ strResult = EntityUtils.toString(respEntity, "UTF-8");
+ if(strResult.equals(Constants.EMPTY)){
+ strResult = "Download failed error is:reslut is null";
+ }
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+ return strResult;
+ }
+ public static String httpPost(String url,String params) {
+ String html="";
+ html = doPost(url,params);
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(5000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ html = doPost(url,params);
+ }
+ return html;
+ }
+ /**
+ * 绕过验证
+ *
+ * @return
+ * @throws NoSuchAlgorithmException
+ * @throws KeyManagementException
+ */
+ public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
+ SSLContext sc = SSLContext.getInstance("SSLv3");
+
+ // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
+ X509TrustManager trustManager = new X509TrustManager() {
+ @Override
+ public void checkClientTrusted(
+ java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
+ String paramString) throws CertificateException {
+ }
+
+ @Override
+ public void checkServerTrusted(
+ java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
+ String paramString) throws CertificateException {
+ }
+
+ @Override
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ };
+
+ sc.init(null, new TrustManager[] { trustManager }, null);
+ return sc;
+ }
+ /**
+ * 模拟请求
+ *
+ * @param url 资源地址
+ * @param map 参数列表
+ * @param encoding 编码
+ * @return
+ * @throws NoSuchAlgorithmException
+ * @throws KeyManagementException
+ * @throws IOException
+ * @throws ClientProtocolException
+ */
+ public static String httpsslGet(String url,Map ... headers) {
+ String html="";
+ CloseableHttpClient client = null;
+ HttpEntity responseEntity = null;
+ CloseableHttpResponse response = null;
+ try {
+ log.debug("DownLoadUtil------------->设置下载相关信息， start....");
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connManager.setMaxTotal(50);
+ connManager.setDefaultMaxPerRoute(10);
+ HttpClients.custom().setConnectionManager(connManager);
+ //设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ // 设置重定向策略
+ LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy();
+ //创建自定义的httpclient对象
+ client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build();
+// CloseableHttpClient client = HttpClients.createDefault();
+
+ HttpGet httpGet = new HttpGet(url);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
+ }
+ log.debug("DownLoadUtil------------->设置下载相关信息， end....");
+ try {
+ int notFundCode = 404;
+ int successCode = 200;
+ log.debug("DownLoadUtil------------->下载执行，start....");
+ httpGet.setConfig(config);
+ response = client.execute(httpGet);
+ log.debug("DownLoadUtil------------->下载执行，end....");
+ // 从响应模型中获取响应实体
+ StatusLine statusLine = response.getStatusLine();
+ log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is start ....");
+ System.out.println("响应状态为:" + response.getStatusLine());
+ responseEntity = response.getEntity();
+ log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is end ....");
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ System.out.println("响应内容长度为:" + responseEntity.getContentLength());
+ }
+ }else if(statusLine.getStatusCode() == notFundCode){
+ html = "页面404，正常结束请求即可
";
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }finally{
+ try {
+ responseEntity.getContent().close();
+ response.close();
+ client.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
+ return html;
+ }
+
+ public static String httpSSLGet(String url,Map ... headers) {
+ String html="";
+ html = httpsslGet(url,headers);
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(30000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ html = httpsslGet(url,headers);
+ }
+ return html;
+ }
+ public static String doPostFrom(String url,Map param,Map ... headers){
+ //设置超时时间
+ int timeout = 15;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+// AuthCache authCache = new BasicAuthCache();
+// authCache.put(proxy, new BasicScheme());
+// HttpClientContext localContext = HttpClientContext.create();
+// localContext.setAuthCache(authCache);
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ httpBuilder.setUserAgent(ua);
+// HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
+// .setDefaultCredentialsProvider(credsProvider).build();
+ HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpPost.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
+ httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
+ httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
+ httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36");
+// httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html");
+ }
+ // 创建请求参数
+ List list = new LinkedList<>();
+ for (String key : param.keySet()) {
+ BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
+ list.add(param1);
+ }
+ // 使用URL实体转换工具
+ String html="";
+ try {
+ UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
+ httpPost.setEntity(entityParam);
+ HttpResponse response = httpClient.execute(httpPost);
+// HttpResponse response = httpClient.execute(httpPost,localContext);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ int notFundCode = 404;
+ int successCode = 200;
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ }
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+
+ return html;
+
+ }
+ public static String httpPostForm(String url,Map params,Map ... headers) {
+ String html="";
+ html = doPostFrom(url,params);
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(5000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ html = doPostFrom(url,params,headers);
+ }
+ return html;
+ }
+
+ public static String dosslPost(String url,String params,Map ... headers) {
+ String html="";
+ CloseableHttpClient client = null;
+ HttpEntity responseEntity = null;
+ CloseableHttpResponse response = null;
+ try {
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ HttpClients.custom().setConnectionManager(connManager);
+ //设置超时时间
+ int timeout = 5;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ //创建自定义的httpclient对象
+ client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
+// CloseableHttpClient client = HttpClients.createDefault();
+ // 2. 创建httppost实例
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpPost.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
+ }
+
+ try {
+ httpPost.setEntity(new StringEntity(params,"utf-8"));
+ response = client.execute(httpPost);
+ int notFundCode = 404;
+ int successCode = 200;
+ // 从响应模型中获取响应实体
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ responseEntity = response.getEntity();
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ System.out.println("响应内容长度为:" + responseEntity.getContentLength());
+ }
+ }else if(statusLine.getStatusCode() == notFundCode){
+ html = "页面404，正常结束请求即可
";
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }finally{
+ try {
+ responseEntity.getContent().close();
+ response.close();
+ client.close();
+ } catch (UnsupportedOperationException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
+ return html;
+ }
+ public static String dosslPostForm(String url,Map param,Map ... headers) {
+ String html="";
+ try {
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ HttpClients.custom().setConnectionManager(connManager);
+ //设置超时时间
+ int timeout = 5;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ //创建自定义的httpclient对象
+ CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
+// CloseableHttpClient client = HttpClients.createDefault();
+ // 2. 创建httppost实例
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpPost.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
+ httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
+ }
+
+ // 创建请求参数
+ List list = new LinkedList<>();
+ for (String key : param.keySet()) {
+ BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
+ list.add(param1);
+ }
+ // 使用URL实体转换工具
+ try {
+ UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
+ httpPost.setEntity(entityParam);
+ HttpResponse response = client.execute(httpPost);
+// HttpResponse response = httpClient.execute(httpPost,localContext);
+ // 从响应模型中获取响应实体
+ int notFundCode = 404;
+ int successCode = 200;
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ }
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+
+
+ return html;
+ }
+ public static String httpSSLPostForm(String url,Map params,Map ...headers) {
+ String html="";
+ try {
+ html = dosslPostForm(url,params,headers);
+ } catch (Exception e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(30000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ try {
+ html = dosslPostForm(url,params,headers);
+ } catch (Exception e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ }
+ return html;
+ }
+ public static String httpSSLPost(String url,String params,Map ...headers) {
+ String html="";
+ try {
+ html = dosslPost(url,params,headers);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(30000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ try {
+ html = dosslPost(url,params,headers);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ }
+ return html;
+ }
+
+ /**
+ * 模拟客户端get请求
+ * @param url 模拟请求得url
+ * @param headers 头部信息，没有可以不传
+ * @return
+ */
+ public static String doGet(String url,Map ... headers){
+ //设置超时时间
+ int timeout = 15;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ httpBuilder.setUserAgent(ua);
+ HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
+ HttpGet httpGet = new HttpGet(url);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ }
+ String html="";
+ try {
+ int notFundCode = 404;
+ int successCode = 200;
+ HttpResponse response = httpClient.execute(httpGet);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ if(html.equals("")){
+ html = "Download failed error is:reslut is null";
+ }
+ }
+ }else if(statusLine.getStatusCode() == notFundCode){
+ html = "页面404，正常结束请求即可
";
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+ return html;
+
+ }
+ public static void main(String[] args) throws Exception {
+
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/utils/EncryptionUtil.java b/src/main/java/com/bfd/assemble/utils/EncryptionUtil.java
new file mode 100644
index 0000000..91fe762
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/utils/EncryptionUtil.java
@@ -0,0 +1,27 @@
+package com.bfd.assemble.utils;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * @author jian.mao
+ * @date 2023年3月10日
+ * @description
+ */
+public class EncryptionUtil {
+ public static String md5(String text) {
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update(text.getBytes());
+ byte[] bytes = md.digest();
+ StringBuilder sb = new StringBuilder();
+ for (byte b : bytes) {
+ sb.append(String.format("%02x", b & 0xff));
+ }
+ return sb.toString();
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/utils/FileUtil.java b/src/main/java/com/bfd/assemble/utils/FileUtil.java
new file mode 100644
index 0000000..6156db2
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/utils/FileUtil.java
@@ -0,0 +1,36 @@
+package com.bfd.assemble.utils;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * 文件工具类
+ * @author jian.mao
+ * @date 2023年7月14日
+ * @description
+ */
+public class FileUtil {
+
+ /**
+ * 数据写入文件
+ * @param Path 文件路径
+ * @param result 数据
+ * @throws IOException
+ */
+ public static void writeFile(String path,String result){
+ try {
+ FileWriter fw = new FileWriter(path,true);
+ fw.write(result+"\n");
+ fw.flush();
+ fw.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/utils/JsonUtil.java b/src/main/java/com/bfd/assemble/utils/JsonUtil.java
new file mode 100644
index 0000000..9f07a21
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/utils/JsonUtil.java
@@ -0,0 +1,32 @@
+package com.bfd.assemble.utils;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.assemble.entity.Constants;
+
+/**
+ * json工具
+ * @author jian.mao
+ * @date 2023年7月10日
+ * @description
+ */
+public class JsonUtil {
+
+ /**
+ * 校验字符串是list/map/str
+ * @param jsonString
+ * @return
+ */
+ public static String checkJsonType(String jsonString) {
+ try {
+ JSONObject.parseObject(jsonString);
+ return Constants.MAP_TYPE;
+ } catch (Exception e) {
+ try {
+ JSONObject.parseArray(jsonString);
+ return Constants.LIST_TYPE;
+ } catch (Exception ex) {
+ return Constants.STRING_TYPE;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/utils/OtherUtils.java b/src/main/java/com/bfd/assemble/utils/OtherUtils.java
new file mode 100644
index 0000000..38f230d
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/utils/OtherUtils.java
@@ -0,0 +1,33 @@
+package com.bfd.assemble.utils;
+
+import java.security.MessageDigest;
+
+/**
+ * 其他工具类
+ * @author jian.mao
+ * @date 2023年9月19日
+ * @description
+ */
+public class OtherUtils {
+
+
+
+ public static String getMd5(String string) {
+ try {
+ MessageDigest md5 = MessageDigest.getInstance("MD5");
+ byte[] bs = md5.digest(string.getBytes("UTF-8"));
+ StringBuilder sb = new StringBuilder(40);
+ for (byte x : bs) {
+ if ((x & 0xff) >> 4 == 0) {
+ sb.append("0").append(Integer.toHexString(x & 0xff));
+ } else {
+ sb.append(Integer.toHexString(x & 0xff));
+ }
+ }
+ return sb.toString();
+ } catch (Exception e) {
+
+ return "nceaform" + System.currentTimeMillis();
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/utils/QueueUtil.java b/src/main/java/com/bfd/assemble/utils/QueueUtil.java
new file mode 100644
index 0000000..8faad5c
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/utils/QueueUtil.java
@@ -0,0 +1,18 @@
+package com.bfd.assemble.utils;
+
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingDeque;
+
+/**
+ * @author:jinming
+ * @className:QueueUtil
+ * @version:1.0
+ * @description:
+ * @Date:2023/7/13 15:00
+ */
+public class QueueUtil {
+
+ public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>();
+
+ public static LinkedBlockingDeque sendQueue = new LinkedBlockingDeque();
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/assemble/utils/SpringBootKafka.java b/src/main/java/com/bfd/assemble/utils/SpringBootKafka.java
new file mode 100644
index 0000000..518454a
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/utils/SpringBootKafka.java
@@ -0,0 +1,46 @@
+package com.bfd.assemble.utils;
+
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.SendResult;
+import org.springframework.stereotype.Component;
+import org.springframework.util.concurrent.ListenableFuture;
+import org.springframework.util.concurrent.ListenableFutureCallback;
+
+/**
+ * @PROJECT_NAME: companybusinesscrawl
+ * @DESCRIPTION:SpringBootKafka 工具类
+ * @AUTHOR: ying.zhao
+ * @DATE: 2023/4/6 11:09
+ */
+@Slf4j
+@Component
+public class SpringBootKafka {
+ @Autowired
+ private KafkaTemplate kafkaTemplate;
+ /**
+ * 自定义topicKafkaTemplate
+ */
+ /**
+ * public static final String TOPIC = "companyBussTest";
+ **/
+ public void send(String topic, String message) {
+ //发送消息
+ ListenableFuture> future = kafkaTemplate.send(topic, message);
+ future.addCallback(new ListenableFutureCallback>() {
+ @Override
+ public void onFailure(Throwable throwable) {
+ //发送失败的处理
+ log.info(topic + " - 生产者 发送消息失败：" + throwable.getMessage());
+ }
+
+ @Override
+ public void onSuccess(SendResult stringObjectSendResult) {
+ //成功的处理
+ log.info("{} - 生产者 发送消息成功：",topic);
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/bfd/assemble/utils/ThrowMessageUtil.java b/src/main/java/com/bfd/assemble/utils/ThrowMessageUtil.java
new file mode 100644
index 0000000..0d137c9
--- /dev/null
+++ b/src/main/java/com/bfd/assemble/utils/ThrowMessageUtil.java
@@ -0,0 +1,23 @@
+package com.bfd.assemble.utils;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+/**
+ * @author jian.mao
+ * @date 2023年3月22日
+ * @description
+ */
+public class ThrowMessageUtil {
+
+ /**
+ * 获取异常信息
+ * @param t
+ * @return
+ */
+ public static String getErrmessage(Throwable t){
+ StringWriter stringWriter=new StringWriter();
+ t.printStackTrace(new PrintWriter(stringWriter,true));
+ return stringWriter.getBuffer().toString();
+ }
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
new file mode 100644
index 0000000..cb97fe1
--- /dev/null
+++ b/src/main/resources/application.yml
@@ -0,0 +1,86 @@
+logging:
+ level:
+ root: info
+ path: ./logs
+ #spring admin boot日志输出配置,需要跟logback-spring.xml配置中日志路径一致
+ file:
+ name: ./logs/resultAssemblyinfo.log
+server:
+ port: 8009
+ servlet:
+ context-path: /resultassembly
+ tomcat:
+ uri-encoding: utf-8
+ max-connections: 20000
+ max-http-form-post-size: 1
+ max-threads: 1000
+spring:
+ application:
+ name: resultassembly
+ kafka:
+ bootstrap-servers: 172.16.12.55:9092,172.16.12.56:9092,172.16.12.57:9092
+ producer:
+ retries: 0
+ #当有多个消息需要被发送到同一个分区时，生产者会把它们放在同一个批次里。该参数指定了一个批次可以使用的内存大小，按照字节数计算。
+ batch-size: 16384
+ # 设置生产者内存缓冲区的大小。
+ buffer-memory: 33554432
+ # 键的序列化方式
+ key-serializer: org.apache.kafka.common.serialization.StringSerializer
+ # 值的序列化方式
+ value-serializer: org.apache.kafka.common.serialization.StringSerializer
+ # acks=0 ： 生产者在成功写入消息之前不会等待任何来自服务器的响应。
+ # acks=1 ： 只要集群的首领节点收到消息，生产者就会收到一个来自服务器成功响应。
+ # acks=all ：只有当所有参与复制的节点全部收到消息时，生产者才会收到一个来自服务器的成功响应。
+ acks: 1
+ consumer:
+ # 自动提交的时间间隔 在spring boot 2.X 版本中这里采用的是值的类型为Duration 需要符合特定的格式，如1S,1M,2H,5D
+ auto-commit-interval: 1S
+ # 该属性指定了消费者在读取一个没有偏移量的分区或者偏移量无效的情况下该作何处理：
+ # latest（默认值）在偏移量无效的情况下，消费者将从最新的记录开始读取数据（在消费者启动之后生成的记录）
+ # earliest ：在偏移量无效的情况下，消费者将从起始位置读取分区的记录
+ auto-offset-reset: earliest
+ # 是否自动提交偏移量，默认值是true,为了避免出现重复数据和数据丢失，可以把它设置为false,然后手动提交偏移量
+ enable-auto-commit: true
+ # 键的反序列化方式
+ key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
+ # 值的反序列化方式
+ value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
+ #消费组
+ group-id: test4
+ #消费者并发线程数
+ concurrency: 4
+ #超时时间
+ max-poll-interval-ms: 60000
+ #listener:
+ # 在侦听器容器中运行的线程数。
+ #concurrency: 5
+ #listner负责ack，每调用一次，就立即commit
+ #ack-mode: manual_immediate
+ #missing-topics-fatal: false
+ boot:
+ admin:
+ client:
+ url: http://172.16.12.55:8001
+ instance:
+ service-base-url: http://10.10.143.85:8009
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: "*"
+ endpoint:
+ health:
+ show-details: always
+ health:
+ elasticsearch:
+ enabled: false
+
+customize-kafka:
+ bootstrap-servers: 172.18.1.119:9992
+ producer:
+ topic: analyze
+task:
+ task-queue-path: ../data/taskQueue.txt
+
diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml
new file mode 100644
index 0000000..37be0c5
--- /dev/null
+++ b/src/main/resources/logback-spring.xml
@@ -0,0 +1,36 @@
+
+
+
+
+
+
+
+
+ true
+
+ ${logging.level}
+
+
+ ${logging.path}/resultAssemblyinfo.log
+
+
+ ${logging.path}/resultAssemblyinfo.log.%d{yyyy-MM-dd}
+ 7
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n
+ UTF-8
+
+
+
+
+
+
+
+
diff --git a/src/test/java/com/bfd/AppTest.java b/src/test/java/com/bfd/AppTest.java
new file mode 100644
index 0000000..7f816c2
--- /dev/null
+++ b/src/test/java/com/bfd/AppTest.java
@@ -0,0 +1,20 @@
+package com.bfd;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+/**
+ * Unit test for simple App.
+ */
+public class AppTest
+{
+ /**
+ * Rigorous Test :-)
+ */
+ @Test
+ public void shouldAnswerWithTrue()
+ {
+ assertTrue( true );
+ }
+}