commit ed10c95aa60b2e1779ce8413a1ae6f005d5d5ac3
Author: 55007 <55007@maojian>
Date: Tue Jan 7 18:06:27 2025 +0800
大模型应用管理
diff --git a/.classpath b/.classpath
new file mode 100644
index 0000000..1a0a8d2
--- /dev/null
+++ b/.classpath
@@ -0,0 +1,40 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..64067b0
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+/logs/
+/target/
+.settings/
diff --git a/.project b/.project
new file mode 100644
index 0000000..098f2c6
--- /dev/null
+++ b/.project
@@ -0,0 +1,23 @@
+
+
+ IntelliModelManager
+
+
+
+
+
+ org.eclipse.jdt.core.javabuilder
+
+
+
+
+ org.eclipse.m2e.core.maven2Builder
+
+
+
+
+
+ org.eclipse.jdt.core.javanature
+ org.eclipse.m2e.core.maven2Nature
+
+
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e7f31f3
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+大预言模型管理
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..c872a67
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,228 @@
+
+
+
+ 4.0.0
+
+ com.bfd
+ IntelliModelManager
+ 0.0.1-SNAPSHOT
+
+ org.springframework.boot
+ spring-boot-starter-parent
+ 2.2.4.RELEASE
+
+ IntelliModelManager
+
+ http://www.example.com
+
+
+ UTF-8
+ 1.8
+ 1.8
+
+
+
+
+ junit
+ junit
+ 4.11
+ test
+
+
+ org.springframework.boot
+ spring-boot-starter-web
+
+
+
+ de.codecentric
+ spring-boot-admin-starter-client
+ 2.2.4
+
+
+ com.google.code.gson
+ gson
+ 2.8.8
+
+
+ org.springframework.boot
+ spring-boot-test
+
+
+
+ org.springframework
+ spring-test
+ 5.0.10.RELEASE
+ test
+
+
+ commons-io
+ commons-io
+ 2.11.0
+
+
+
+ com.alibaba
+ fastjson
+ 2.0.17
+
+
+
+ com.mchange
+ c3p0
+ 0.9.5.5
+
+
+
+ com.squareup.okhttp3
+ okhttp
+ 4.9.3
+
+
+ org.apache.httpcomponents
+ httpclient
+ 4.5.3
+
+
+ commons-lang
+ commons-lang
+ 2.6
+
+
+
+ org.jetbrains.kotlin
+ kotlin-reflect
+ 1.6.21
+ runtime
+
+
+
+
+ org.apache.logging.log4j
+ log4j-api
+ 2.14.1
+
+
+ org.apache.logging.log4j
+ log4j-core
+ 2.14.1
+
+
+
+ org.projectlombok
+ lombok
+
+
+ cn.hutool
+ hutool-all
+ 5.8.5
+
+
+ junit
+ junit
+
+
+
+ p6spy
+ p6spy
+ 3.9.0
+
+
+
+ commons-collections
+ commons-collections
+ 3.2.2
+
+
+
+
+
+
+
+
+
+ maven-clean-plugin
+ 3.1.0
+
+
+
+ maven-resources-plugin
+ 3.0.2
+
+
+ maven-compiler-plugin
+ 3.8.0
+
+
+ maven-surefire-plugin
+ 2.22.1
+
+
+ maven-jar-plugin
+ 3.0.2
+
+
+ maven-install-plugin
+ 2.5.2
+
+
+ maven-deploy-plugin
+ 2.8.2
+
+
+
+ maven-site-plugin
+ 3.7.1
+
+
+ maven-project-info-reports-plugin
+ 3.0.0
+
+
+
+
+ org.springframework.boot
+ spring-boot-maven-plugin
+
+ com.bfd.Application
+ ZIP
+
+
+ ${project.groupId}
+ ${project.artifactId}
+
+
+
+
+
+
+ repackage
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+ 3.1.1
+
+
+ copy
+ package
+
+ copy-dependencies
+
+
+ jar
+ jar
+ runtime
+ ${project.build.directory}/libs
+
+
+
+
+
+
+
+
diff --git a/src/main/java/com/bfd/Application.java b/src/main/java/com/bfd/Application.java
new file mode 100644
index 0000000..2db3df6
--- /dev/null
+++ b/src/main/java/com/bfd/Application.java
@@ -0,0 +1,24 @@
+package com.bfd;
+
+
+
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+/**
+ * 主入口
+ *
+ * @author jian.mao
+ * @date 2023年7月4日
+ * @description
+ */
+@SpringBootApplication
+@EnableScheduling
+public class Application {
+
+ public static void main(String[] args) {
+ SpringApplication.run(Application.class, args);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/cache/ConfigCache.java b/src/main/java/com/bfd/cache/ConfigCache.java
new file mode 100644
index 0000000..c510cd4
--- /dev/null
+++ b/src/main/java/com/bfd/cache/ConfigCache.java
@@ -0,0 +1,37 @@
+package com.bfd.cache;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingDeque;
+
+/**
+ * @author jian.mao
+ * @date 2022年11月11日
+ * @description 静态变量类
+ */
+@Slf4j
+public class ConfigCache {
+
+ /**启动条件**/
+ public static boolean isStart = true;
+ /*****任务队列*****/
+ public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>();
+ /****视频生成队列***/
+ public static LinkedBlockingDeque> videoTaskQueue = new LinkedBlockingDeque>();
+
+
+ /**
+ * 队列录入任务
+ * @param queue
+ * @param task
+ */
+ public static void putQueue(LinkedBlockingDeque> queue,Map task){
+ //next app 写入队列准备调出
+ try {
+ queue.put(task);
+ } catch (InterruptedException e) {
+ log.error("队列写入data失败---");
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/controller/ManagementController.java b/src/main/java/com/bfd/controller/ManagementController.java
new file mode 100644
index 0000000..8973dc8
--- /dev/null
+++ b/src/main/java/com/bfd/controller/ManagementController.java
@@ -0,0 +1,37 @@
+package com.bfd.controller;
+
+import javax.annotation.Resource;
+
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.bfd.service.ManagementService;
+
+/**
+ * 模型管理控制器
+ * @author jian.mao
+ * @date 2024年7月29日
+ * @description
+ */
+@Controller
+@RequestMapping("/model")
+public class ManagementController {
+ @Resource
+ private ManagementService managementService;
+
+ @PostMapping("/modelSelect")
+ @ResponseBody
+ public String modelSelect(@RequestBody String dataJson){
+ String response = managementService.modelSelect(dataJson);
+ return response;
+ }
+ @PostMapping("/videoModelSelect")
+ @ResponseBody
+ public String videoModelSelect(@RequestBody String dataJson){
+ String response = managementService.videoModelSelect(dataJson);
+ return response;
+ }
+}
diff --git a/src/main/java/com/bfd/entity/Constants.java b/src/main/java/com/bfd/entity/Constants.java
new file mode 100644
index 0000000..ca32e1d
--- /dev/null
+++ b/src/main/java/com/bfd/entity/Constants.java
@@ -0,0 +1,183 @@
+package com.bfd.entity;
+
+
+/**
+ * 常量实体类
+ * @author jian.mao
+ * @date 2022年11月15日
+ * @description
+ */
+public class Constants {
+
+ /*************************蓝图常量key名称*********************************/
+ public final static String SCHEDULING = "scheduling";
+ public final static String TYPE = "type";
+ public final static String INTERVAL = "interval";
+ public final static String CREATED = "created";
+ public final static String LAST_EDIT = "last_edit";
+ public final static String BLUEPRINT_ID = "blueprint_id";
+ public final static String BLUEPRINTID = "blueprintId";
+ public final static String BLUEPRINT_NAME = "name";
+ public final static String SCENARIO = "scenario";
+ public final static String AUTOCOMMITTRIGGERLAST = "autoCommitTriggerLast";
+ public final static String FRESHVARIABLES = "freshVariables";
+ public final static String AUTOCOMMIT = "autoCommit";
+ public final static String MAXERRORS = "maxErrors";
+ public final static String DATALOSS = "dataloss";
+ public final static String POSITION = "position";
+ public final static String SCENES_ID = "scenes_id";
+ public final static String SCENESID = "scenesId";
+ public final static String MULTI_BRANCH = "multi_branch";
+
+ public final static String SINGLE = "single";
+ /**已重试次数**/
+ public final static String ERROR_TIME = "error_time";
+ public final static String PREVIOUS_RESULT = "previous_result";
+
+ /****数据id*****/
+ public final static String BUSINESSKEY = "businessKey";
+
+
+ /*************************metadata常量key名称*********************************/
+ public final static String LABEL_COL = "label_col";
+ public final static String LABEL = "label";
+ public final static String USER = "user";
+ public final static String ADMIN = "admin";
+ public final static String ADDRESS = "address";
+ public final static String DATASOURCE = "datasource";
+ public final static String INDEX = "index";
+
+ /*************************app常量key名称*********************************/
+ public final static String APPS = "apps";
+ public final static String TRANSFER_ID = "transfer_id";
+ public final static String MODULE = "module";
+ public final static String VERSION = "version";
+ public final static String METADATA = "metadata";
+ public final static String APP_NAME = "name";
+ public final static String DESCRIBE = "describe";
+ public final static String NEXT_APP_ID = "next_app_id";
+ public final static String EDGE_ID = "edge_id";
+ public final static String START_ID = "start_id";
+ public final static String END_ID = "end_id";
+
+ public final static String WAIT_CONDITION = "wait_condition";
+ public final static String START_TAG = "start_tag";
+
+ /*************************module类型*********************************/
+ public final static String FILE = "file";
+ public final static String OCR = "OCR";
+ public final static String FILTER = "Filter";
+ public final static String CHATGPT = "ChatGPT";
+ public final static String MYSQL = "mysql";
+
+ /*************************other类型*********************************/
+ public final static String UNDERLINE = "_";
+ public final static String RESULT_TOPIC = null;
+ public static final String EMPTY = "";
+ public static final String HTTP = "http";
+ public static final String REQUEST_ERROR_MESSAGE = "Download failed error is";
+ public static final String REQUEST_RESULT = "result";
+ public static final String REQUEST_RESULT_RESULTS = "results";
+ public static final String MAP_TYPE = "Map";
+ public static final String LIST_TYPE = "List";
+ public static final String STRING_TYPE = "String";
+ public static final String DOCUMENT_TYPE = "doc";
+ public static final String FILTER_ZH = "过滤器";
+
+ public static final String JSON_SELE_SYMBOL = "$.";
+ public static final String LEFT_BRACKETS = "[";
+ public static final String RIGTH_BRACKETS = "]";
+ public static final String TASKTYPE = "taskType";
+ public static final Integer USER_TYPE = 1;
+ public static final Integer KEYWORD_TYPE = 0;
+ public static final Integer DETAIL_TYPE = 2;
+ public static final String CID = "cid";
+ public static final String SITETYPE = "siteType";
+ public static final Integer DEFULT_SUBJECTID = 304864;
+ public static final Integer DEFULT_CRAWLCYCLICITYTIME = 1440;
+ public static final String CRAWLENDTIME = "crawlEndTime";
+ public static final String CRAWLSTARTTIME = "crawlStartTime";
+ public static final String CRAWLPAGETYPES = "crawlPageTypes";
+ public static final String APPID = "113ic";
+ public static final String APP_ID = "appId";
+ public final static String ID = "id";
+ public static final Integer DEFULT_CRAWLPERIODHOUR = 24;
+ public static final String CREATEUSERID = "662015832180933762";
+ public static final String CRAWL_ADD_URL = "https://caiji.percent.cn/api/crawl/remote/task/save";
+ public static final String CRAWLKEYWORD = "crawlKeyword";
+ public static final String ATTACHTAG = "attachTag";
+ public static final String ATTACHTAG_VALUE = "analyze";
+ public static final String KEYWORD = "keyword";
+ public static final String SITEID = "siteId";
+ public static final String RESULTS = "results";
+ public static final String RESULT = "result";
+ public static final String CRAWLDATAFLAG = "crawlDataFlag";
+ public static final String CRAWLDATAFLAG_PREFIX = "\"crawlDataFlag\":\"keyword:";
+ public static final String TID = "tid";
+ public static final Long TIME_OUT = 1800000L;
+ public static final String ATTR = "attr";
+ public static final String HASVIDEO = "hasVideo";
+ public static final String CRAWL_END_MARK = "crawl_end_mark";
+ public static final String CRAWL_END_MESSAGE = "crawl_end_message";
+ public static final String CRAWL_END_MESSAGE_VALUE = "数据采集完成";
+ public static final String SUBJECTID = "subjectId";
+ public static final String TASKID = "taskId";
+ public static final int SUCCESS_CODE = 200;
+ public static final String WEB_URL_SUFFIX = "/api/aogeo/api/cda/caiji/status";
+ public static final String STATUS = "status";
+ /************************redis*************************************/
+ public static final String LOCK_KEY = "myLock";
+ public static final long LOCK_EXPIRE_TIME = 300000;
+
+ /************************应用参数*************************************/
+ public static final String CODE = "code";
+ public static final String MESSAGE = "message";
+ public static final String INPUT = "input";
+ public static final String OUTPUT = "output";
+ public static final String FORM = "form";
+ public static final String FIELD = "field";
+ public static final String VALUE = "value";
+ public static final String DATA = "data";
+ public static final String COLON_EN = ":";
+ public static final String DATABASE = "database";
+ public static final String TABLE = "table";
+ public static final String USERNAME = "username";
+ public static final String PASSWORD = "password";
+ public static final String PORT = "port";
+ public static final String HOSTNAME = "hostname";
+ public static final String DATATYPE = "dataType";
+ public static final String RULES = "rules";
+ public static final String GENID = "genId";
+ public static final String KEY = "key";
+ public static final String DATAID = "dataId";
+ public static final String APP_CODE = "app_code";
+
+ /***************文件相关参数常量***********************/
+ public static final String READCONTENT = "readContent";
+ public static final String CONTENT = "content";
+ public static final String JPG = "jpg";
+ public static final String PNG = "png";
+ public static final String JPEG = "jpeg";
+ public static final String XLS = "xls";
+ public static final String XLSX = "xlsx";
+ public static final String PDF = "pdf";
+ public static final String DOC = "doc";
+ public static final String DOCX = "docx";
+ public static final String PPT = "ppt";
+ public static final String PPTX = "pptx";
+ public static final String MP4 = "mp4";
+ public static final String EXT = "ext";
+ public static final String FILES = "files";
+ public static final String PATH = "path";
+ public static final String TITLE = "title";
+ public static final String DATAPROCESSID = "dataProcessId";
+ public static final String LLM = "llm";
+ public static final String GPT_VALUE = "1";
+ public static final String GLM_VALUE = "2";
+ public static final String QWEN_VALUE = "3";
+ public static final String MINIMAX_VALUE = "4";
+ public static final String DOUBAO_VALUE = "5";
+ public static final String MODELADDRESS = "modelAddress";
+ public static final String AUTHORIZATIONS = "authorizations";
+ public static final String AUTHORIZATION = "authorization";
+}
diff --git a/src/main/java/com/bfd/handler/MainHandler.java b/src/main/java/com/bfd/handler/MainHandler.java
new file mode 100644
index 0000000..f29a14f
--- /dev/null
+++ b/src/main/java/com/bfd/handler/MainHandler.java
@@ -0,0 +1,132 @@
+package com.bfd.handler;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import javax.annotation.Resource;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.io.FileUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
+import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.cache.ConfigCache;
+import com.bfd.process.ModelSelectProcess;
+import com.bfd.process.VideoModelSelectProcess;
+import com.bfd.utils.FileUtil;
+
+
+/**
+ * 启动处理入口
+ * @author jian.mao
+ * @date 2023年11月3日
+ * @description
+ */
+
+@Component
+@Order(value = 1)
+@Slf4j
+public class MainHandler implements ApplicationRunner {
+
+ @Value("${task.task-queue-path}")
+ private String taskPath;
+ @Value("${task.video-task-queue-path}")
+ private String videoTaskPath;
+ @Resource
+ private ModelSelectProcess modelSelectProcess;
+ @Resource
+ private VideoModelSelectProcess videoModelSelectProcess;
+ @Override
+ public void run(ApplicationArguments args) throws Exception {
+ new Thread(modelSelectProcess).start();;
+ log.info("开启模型任务分发执行线程-----");
+ new Thread(videoModelSelectProcess).start();;
+ log.info("开启视频生成模型任务分发执行线程-----");
+ //停止处理
+ waitDown();
+ //启动加载缓存任务
+ readTask(taskPath,ConfigCache.taskQueue);
+ readTask(videoTaskPath,ConfigCache.videoTaskQueue);
+ }
+
+
+
+
+
+ @SuppressWarnings("unchecked")
+ public static void readTask(String path,LinkedBlockingDeque> queue){
+ File file = new File(path);
+ if(file.exists()){
+ List tasks = null;
+ try {
+ tasks = FileUtils.readLines(file,"UTF-8");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ for (String taskStr : tasks) {
+ Map task = JSONObject.parseObject(taskStr);
+ try {
+ queue.put(task);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ file.delete();
+ }
+ }
+ /**
+ * 结束触发钩子
+ */
+ public void waitDown() {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ // 停止线程
+ ConfigCache.isStart = false;
+ log.info("stop-------");
+ writeTsskToFile();
+ }
+ });
+ }
+
+
+ /**
+ * 任务持久化到硬盘
+ */
+ public void writeTsskToFile(){
+ while(true){
+ if(ConfigCache.taskQueue.size() > 0 ){
+ try {
+ Map task = ConfigCache.taskQueue.take();
+ FileUtil.writeFile(taskPath, JSONObject.toJSONString(task));
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }else{
+ log.info("taskQueue write is file end");
+ break;
+ }
+ }
+ while(true){
+ if(ConfigCache.videoTaskQueue.size() > 0 ){
+ try {
+ Map task = ConfigCache.videoTaskQueue.take();
+ FileUtil.writeFile(videoTaskPath, JSONObject.toJSONString(task));
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }else{
+ log.info("taskQueue write is file end");
+ break;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/process/ModelSelectProcess.java b/src/main/java/com/bfd/process/ModelSelectProcess.java
new file mode 100644
index 0000000..98ff9d3
--- /dev/null
+++ b/src/main/java/com/bfd/process/ModelSelectProcess.java
@@ -0,0 +1,74 @@
+package com.bfd.process;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.stereotype.Component;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.cache.ConfigCache;
+import com.bfd.entity.Constants;
+import com.bfd.utils.DateUtil;
+import com.bfd.utils.DownLoadUtil;
+
+
+/**1
+ * @author jian.mao
+ * @date 2024年7月29日
+ * @description
+ */
+@Component
+@Slf4j
+public class ModelSelectProcess implements Runnable{
+
+ private Random random = new Random();
+ @Override
+ public void run() {
+ while (ConfigCache.isStart){
+ try {
+ Map task = null;
+ task = ConfigCache.taskQueue.take();
+ log.info("任务队列长度：{}", Integer.valueOf(ConfigCache.taskQueue.size()));
+ Map input = (Map) task.get(Constants.INPUT);
+ Map modelAddress = (Map) input.get(Constants.MODELADDRESS);
+ Map authorizations = (Map) input.get(Constants.AUTHORIZATIONS);
+ String llm = (String) input.get(Constants.LLM);
+ String address = (String) modelAddress.get(llm);
+ List authorizationList = (List) authorizations.get(llm);
+ String authorization = authorizationList.get(random.nextInt(authorizationList.size()));
+ input.put(Constants.AUTHORIZATION, authorization);
+ String response = null;
+ if(llm.equals(Constants.GPT_VALUE)){
+ log.info("chatgpt 智能模型任务触发-----");
+ response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task));
+ }else if(llm.equals(Constants.GLM_VALUE)){
+ log.info("智谱 智能模型任务触发-----");
+ response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task));
+ }else if(llm.equals(Constants.QWEN_VALUE)){
+ log.info("本地 智能模型任务触发-----");
+ response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task));
+ }else if(llm.equals(Constants.MINIMAX_VALUE)){
+ log.info("minimax 智能模型任务触发-----");
+ response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task));
+ }else if(llm.equals(Constants.DOUBAO_VALUE)){
+ log.info("doubao 智能模型任务触发-----");
+ response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task));
+ }else{
+ log.error("未知模型，不处理");
+ }
+ if(response.contains(Constants.REQUEST_ERROR_MESSAGE)){
+ log.error("接口请求异常，address：{}，errormessage：{}",address,response);
+ }else{
+ log.info("模型接口请求正常！接口地址{}，接口信息：{}",address,response);
+ }
+ } catch (Throwable e) {
+ log.error("结果组装异常，", e);
+ log.info("数据流转至下游-------");
+ }
+
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/process/VideoModelSelectProcess.java b/src/main/java/com/bfd/process/VideoModelSelectProcess.java
new file mode 100644
index 0000000..db64e1b
--- /dev/null
+++ b/src/main/java/com/bfd/process/VideoModelSelectProcess.java
@@ -0,0 +1,57 @@
+package com.bfd.process;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.stereotype.Component;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.cache.ConfigCache;
+import com.bfd.entity.Constants;
+import com.bfd.utils.DateUtil;
+import com.bfd.utils.DownLoadUtil;
+
+
+/**1
+ * @author jian.mao
+ * @date 2024年7月29日
+ * @description
+ */
+@Component
+@Slf4j
+public class VideoModelSelectProcess implements Runnable{
+
+ private Random random = new Random();
+ @Override
+ public void run() {
+ while (ConfigCache.isStart){
+ try {
+ Map task = null;
+ task = ConfigCache.videoTaskQueue.take();
+ log.info("任务队列长度：{}", Integer.valueOf(ConfigCache.videoTaskQueue.size()));
+ Map input = (Map) task.get(Constants.INPUT);
+ Map modelAddress = (Map) input.get(Constants.MODELADDRESS);
+ Map authorizations = (Map) input.get(Constants.AUTHORIZATIONS);
+ String llm = (String) input.get(Constants.LLM);
+ String address = (String) modelAddress.get(llm);
+ List authorizationList = (List) authorizations.get(llm);
+ String authorization = authorizationList.get(random.nextInt(authorizationList.size()));
+ input.put(Constants.AUTHORIZATION, authorization);
+ String response = null;
+ if(llm.equals(Constants.MINIMAX_VALUE)){
+ log.info("minimax 视频生成模型任务触发-----");
+ response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task));
+ }else{
+ log.info("视频生成模型接口请求正常！接口地址{}，接口信息：{}",address,response);
+ }
+ } catch (Throwable e) {
+ log.error("结果组装异常，", e);
+ log.info("数据流转至下游-------");
+ }
+
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/service/ManagementService.java b/src/main/java/com/bfd/service/ManagementService.java
new file mode 100644
index 0000000..20bd39b
--- /dev/null
+++ b/src/main/java/com/bfd/service/ManagementService.java
@@ -0,0 +1,25 @@
+package com.bfd.service;
+
+
+/**
+ *
+ * @author jian.mao
+ * @date 2024年7月29日
+ * @description
+ */
+public interface ManagementService {
+
+ /**
+ * 模型选择
+ * @param dataJson
+ * @return
+ */
+ public String modelSelect(String dataJson);
+
+ /**
+ * 视频模型选择
+ * @param dataJson
+ * @return
+ */
+ public String videoModelSelect(String dataJson);
+}
diff --git a/src/main/java/com/bfd/service/impl/ManagementServiceImpl.java b/src/main/java/com/bfd/service/impl/ManagementServiceImpl.java
new file mode 100644
index 0000000..a74e65e
--- /dev/null
+++ b/src/main/java/com/bfd/service/impl/ManagementServiceImpl.java
@@ -0,0 +1,77 @@
+package com.bfd.service.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.stereotype.Service;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.cache.ConfigCache;
+import com.bfd.entity.Constants;
+import com.bfd.service.ManagementService;
+
+/**
+ *
+ * @author jian.mao
+ * @date 2024年7月29日
+ * @description
+ */
+@Service
+@Slf4j
+public class ManagementServiceImpl implements ManagementService {
+
+ @Override
+ public String modelSelect(String dataJson) {
+ // TODO Auto-generated method stub
+ Map response = new HashMap<>(16);
+ int code = 200;
+ String message = "success";
+ Map task = null;
+ try {
+ task = JSONObject.parseObject(dataJson);
+ } catch (Exception e) {
+ log.error("参数结构不合法，",e);
+ code = 100010;
+ message = "参数不合法";
+ }
+ //写入队列
+ try {
+ ConfigCache.taskQueue.put(task);
+ } catch (InterruptedException e) {
+ log.error("任务写入等待队列异常，",e);
+ code = 100011;
+ message = "任务写入等待队列失败";
+ }
+ response.put(Constants.CODE,code);
+ response.put(Constants.MESSAGE,message);
+ return JSONObject.toJSONString(response);
+ }
+ @Override
+ public String videoModelSelect(String dataJson) {
+ // TODO Auto-generated method stub
+ Map response = new HashMap<>(16);
+ int code = 200;
+ String message = "success";
+ Map task = null;
+ try {
+ task = JSONObject.parseObject(dataJson);
+ } catch (Exception e) {
+ log.error("参数结构不合法，",e);
+ code = 100010;
+ message = "参数不合法";
+ }
+ //写入队列
+ try {
+ ConfigCache.videoTaskQueue.put(task);
+ } catch (InterruptedException e) {
+ log.error("视频任务写入等待队列异常，",e);
+ code = 100011;
+ message = "视频任务写入等待队列失败";
+ }
+ response.put(Constants.CODE,code);
+ response.put(Constants.MESSAGE,message);
+ return JSONObject.toJSONString(response);
+ }
+}
diff --git a/src/main/java/com/bfd/utils/DataUtil.java b/src/main/java/com/bfd/utils/DataUtil.java
new file mode 100644
index 0000000..7daabf0
--- /dev/null
+++ b/src/main/java/com/bfd/utils/DataUtil.java
@@ -0,0 +1,63 @@
+package com.bfd.utils;
+
+import java.util.Map;
+
+import lombok.extern.slf4j.Slf4j;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.JSONPath;
+import com.bfd.entity.Constants;
+
+/**
+ * @author:jinming
+ * @className:DataUtil
+ * @version:1.0
+ * @description: 获取dataValue的值
+ * @Date:2023/11/1 9:54
+ */
+@Slf4j
+public class DataUtil {
+ /**
+ *
+ * @param key 传入的key
+ * @param dataMap 数据map
+ * @return 根据传入的参数进行判断解析，返回正确的dataValue
+ */
+ public static Object getValue(String key, Map dataMap) {
+ try {
+ //公式为空直接就返回
+ if(key.equals(Constants.EMPTY)){
+ return Constants.EMPTY;
+ }
+ Object dataValue;
+ String isJson = "#json#";
+ if (key.contains(isJson)) {
+ //进行第一次拆分，获取#json#前面的部分
+ String[] keySplit = key.split(isJson);
+ String firstDataKey = keySplit[0];
+ String[] firstDataKeySplit = firstDataKey.split(":");
+ //取出前半部分对应的JSON数据并转换为JSONObject
+ String dataJson = (String) dataMap.get(firstDataKeySplit[0]);
+ JSONObject dataJsonObject = JSON.parseObject(dataJson);
+ //根据key的后半部分取出对应JSONObject中的值
+ String firstDataKeyJson = (String) JSONPath.eval(dataJsonObject, firstDataKeySplit[1]);
+ String secDataKey = keySplit[1];
+ JSONObject firstDataJsonObject = JSON.parseObject(firstDataKeyJson);
+ dataValue = JSONPath.eval(firstDataJsonObject, secDataKey);
+ return dataValue;
+ }
+ String[] keySplit = key.split(":");
+ String jsonPath = keySplit[1];
+ String dataJson = (String) dataMap.get(keySplit[0]);
+ JSONObject dataJsonObject = JSON.parseObject(dataJson);
+ dataValue = JSONPath.eval(dataJsonObject, jsonPath);
+ return dataValue;
+ } catch (Exception e) {
+ // TODO: handle exception
+ log.error("jsonpath公式取值异常，",e);
+ return null;
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/utils/DateUtil.java b/src/main/java/com/bfd/utils/DateUtil.java
new file mode 100644
index 0000000..6e238db
--- /dev/null
+++ b/src/main/java/com/bfd/utils/DateUtil.java
@@ -0,0 +1,177 @@
+package com.bfd.utils;
+
+
+import java.math.BigInteger;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+
+import lombok.extern.slf4j.Slf4j;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+
+/**
+ * 日期工具类
+ *
+ * @author jian.mao
+ * @date 2022年11月15日
+ * @description
+ */
+@Slf4j
+public class DateUtil {
+
+ /**
+ * @return
+ */
+ public static String getTimeStrForNow() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH");
+ return sdf.format(new Date());
+ }
+
+
+ public static String getTimeStrForDay(long time) {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+
+ return sdf.format(new Date(time * 1000));
+ }
+
+ public static String getTimeStrForDay() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+
+ return sdf.format(new Date());
+ }
+
+
+ public static String getDateTime() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String time = sdf.format(new Date());
+ return time;
+ }
+
+ public static String getDateTime(Long timestap) {
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String time = sdf.format(new Date(timestap));
+ return time;
+ }
+
+ public static String getDate(Long timestap) {
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+ String time = sdf.format(new Date(timestap));
+ return time;
+ }
+
+ public static String getDateTimeForMonth() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM");
+ String time = sdf.format(new Date());
+ return time;
+ }
+
+ /**
+ * 休眠
+ *
+ * @param millis 毫秒
+ */
+ public static void sleep(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * 1. @Description:时间戳转时间
+ * 2. @Author: ying.zhao
+ * 3. @Date: 2023/3/28
+ */
+
+ public static String timestampToDate(String time) {
+ int thirteen = 13;
+ int ten = 10;
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+// if (time.length() == thirteen) {
+ if (time.length() > ten) {
+ return sdf.format(new Date(Long.parseLong(time)));
+ } else {
+ return sdf.format(new Date(Integer.parseInt(time) * 1000L));
+ }
+ }
+
+ public static String parseCreated(String jsonTime){
+ String formattedDateTime = getDateTime();
+ try {
+ // 使用fastjson解析JSON数据
+ JSONObject jsonObject = JSON.parseObject(jsonTime);
+ // 获取日期和时间的值
+ JSONObject dateObject = jsonObject.getJSONObject("date");
+ int day = dateObject.getIntValue("day");
+ int month = dateObject.getIntValue("month");
+ int year = dateObject.getIntValue("year");
+
+ JSONObject timeObject = jsonObject.getJSONObject("time");
+ int hour = timeObject.getIntValue("hour");
+ int minute = timeObject.getIntValue("minute");
+ int second = timeObject.getIntValue("second");
+
+ // 创建LocalDateTime对象
+ LocalDateTime dateTime = LocalDateTime.of(year, month, day, hour, minute, second);
+
+ // 定义日期时间格式化器
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+ // 格式化日期时间
+ formattedDateTime = dateTime.format(formatter);
+ } catch (Exception e) {
+ log.info("日期转换失败：{}",e);
+ }
+ return formattedDateTime;
+ }
+
+ /**
+ * 字符串转换日期
+ * @param format
+ * @param date
+ * @return
+ */
+ public static Date strToDate(String format,String date){
+ SimpleDateFormat sdf = new SimpleDateFormat(format);
+ if (date == null || date.equals("")){
+ return new Date();
+ }else{
+ Date ru = null;
+ try {
+ ru = sdf.parse(date);
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+ return ru;
+ }
+ }
+ /**
+ * 日期格式话
+ * @param format 日期格式
+ * @param dater 要转换的日期,默认当前时间
+ * @return
+ */
+ public static String FormatDate(String format,Date date){
+ String fromatDate = null;
+ SimpleDateFormat sdf = new SimpleDateFormat(format);
+ if (date == null){
+ fromatDate = sdf.format(new Date());
+ }else{
+ fromatDate = sdf.format(date);
+ }
+ return fromatDate;
+ }
+ public static void main(String[] args) {
+ String time = timestampToDate("955814400000");
+ System.out.println(time);
+ }
+}
diff --git a/src/main/java/com/bfd/utils/DownLoadUtil.java b/src/main/java/com/bfd/utils/DownLoadUtil.java
new file mode 100644
index 0000000..96847b4
--- /dev/null
+++ b/src/main/java/com/bfd/utils/DownLoadUtil.java
@@ -0,0 +1,1007 @@
+package com.bfd.utils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import okhttp3.MediaType;
+import okhttp3.MultipartBody;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.StatusLine;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.AuthCache;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.config.SocketConfig;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.LaxRedirectStrategy;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.bfd.entity.Constants;
+
+
+
+
+
+
+
+
+/**
+ * 下载工具类
+ * @author jian.mao
+ * @date 2023年9月19日
+ * @description
+ */
+public class DownLoadUtil {
+
+ private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36";
+ private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class);
+ /** 代理服务器(产品官网 www.16yun.cn) **/
+ final static String PROXYHOST = "u270.40.tp.16yun.cn";
+ final static Integer PROXYPORT = 6448;
+ /** 代理验证信息 **/
+ final static String PROXYUSER = "16HFBVJC";
+ final static String PROXYPASS = "897944";
+
+ private static PoolingHttpClientConnectionManager cm = null;
+ private static HttpRequestRetryHandler httpRequestRetryHandler = null;
+ private static HttpHost proxy = null;
+
+ private static CredentialsProvider credsProvider = null;
+ private static RequestConfig reqConfig = null;
+ private static OkHttpClient okHttpClient;
+
+ private static OkHttpClient getOkHttpClient() {
+ if (okHttpClient == null) {
+ okHttpClient = new OkHttpClient();
+ }
+ return okHttpClient;
+ }
+ static {
+ ConnectionSocketFactory plainsf = PlainConnectionSocketFactory
+ .getSocketFactory();
+ LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory
+ .getSocketFactory();
+
+ Registry registry = RegistryBuilder.create().register("http", plainsf)
+ .register("https", sslsf).build();
+
+ cm = new PoolingHttpClientConnectionManager(registry);
+ cm.setMaxTotal(20);
+ cm.setDefaultMaxPerRoute(5);
+
+ proxy = new HttpHost(PROXYHOST, PROXYPORT, "https");
+
+ credsProvider = new BasicCredentialsProvider();
+ credsProvider.setCredentials(AuthScope.ANY,
+ new UsernamePasswordCredentials(PROXYUSER, PROXYPASS));
+
+ reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000)
+ .setConnectTimeout(5000).setSocketTimeout(5000)
+ .setExpectContinueEnabled(false)
+ .setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build();
+ }
+
+ /**
+ * 模拟客户端get请求
+ *
+ * @param url
+ * 模拟请求得url
+ * @param headers
+ * 头部信息，没有可以不传
+ * @return
+ */
+ @SafeVarargs
+ public static String proxyDoGet(String url, Map... headers) {
+ // 设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom()
+ .setConnectTimeout(timeout * 1000)
+ .setConnectionRequestTimeout(timeout * 1000)
+ .setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(timeout * 1000)
+ .setTcpNoDelay(true).build();
+ AuthCache authCache = new BasicAuthCache();
+ authCache.put(proxy, new BasicScheme());
+ HttpClientContext localContext = HttpClientContext.create();
+ localContext.setAuthCache(authCache);
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ CloseableHttpClient httpClient = httpBuilder
+ .setDefaultSocketConfig(socketConfig)
+ .setDefaultRequestConfig(config)
+ .setDefaultCredentialsProvider(credsProvider).build();
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setConfig(reqConfig);
+ if (headers != null && headers.length > 0) {
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key, tempHeaders.get(key).toString());
+ }
+ } else {
+ httpGet.setHeader("Accept",
+ "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
+ }
+ CloseableHttpResponse response = null;
+ String html = "";
+ int notFundCode = 404;
+ int successCode = 200;
+ try {
+ response = httpClient.execute(httpGet, localContext);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if (statusLine.getStatusCode() == successCode) {
+ if (responseEntity != null) {
+ html = EntityUtils.toString(responseEntity, "utf-8");
+ System.out.println("响应内容长度为:"
+ + responseEntity.getContentLength());
+ // 下载结果为空不正常
+ if (html.equals(Constants.EMPTY)) {
+ html = "Download failed error is:reslut is null";
+ }
+ }
+ } else if (statusLine.getStatusCode() == notFundCode) {
+ html = "页面404，正常结束请求即可
";
+ } else {
+ throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:reslut is null";
+ }finally{
+ try {
+ response.close();
+ httpClient.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ return html;
+
+ }
+
+
+ public static String httpsslProxyGet(String url, Map... headers) throws Exception {
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connManager.setMaxTotal(50);
+ connManager.setDefaultMaxPerRoute(10);
+ HttpClients.custom().setConnectionManager(connManager);
+ // 设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom()
+ .setConnectTimeout(timeout * 1000)
+ .setConnectionRequestTimeout(timeout * 1000)
+ .setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(timeout * 1000)
+ .setTcpNoDelay(true).build();
+ AuthCache authCache = new BasicAuthCache();
+ authCache.put(proxy, new BasicScheme());
+ HttpClientContext localContext = HttpClientContext.create();
+ localContext.setAuthCache(authCache);
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ CloseableHttpClient httpClient = httpBuilder
+ .setConnectionManager(connManager)
+ .setDefaultSocketConfig(socketConfig)
+ .setDefaultRequestConfig(config)
+ .setDefaultCredentialsProvider(credsProvider).build();
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setConfig(reqConfig);
+ if (headers != null && headers.length > 0) {
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key, tempHeaders.get(key).toString());
+ }
+ } else {
+ httpGet.setHeader("Accept",
+ "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
+ }
+ CloseableHttpResponse response = null;
+ String html = "";
+ int notFundCode = 404;
+ int successCode = 200;
+ try {
+ response = httpClient.execute(httpGet, localContext);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if (statusLine.getStatusCode() == successCode) {
+ if (responseEntity != null) {
+ html = EntityUtils.toString(responseEntity, "utf-8");
+ System.out.println("响应内容长度为:"
+ + responseEntity.getContentLength());
+ // 下载结果为空不正常
+ if (html.equals(Constants.EMPTY)) {
+ html = "Download failed error is:reslut is null";
+ }
+ }
+ } else if (statusLine.getStatusCode() == notFundCode) {
+ html = "页面404，正常结束请求即可
";
+ } else {
+ throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:reslut is null";
+ }finally{
+ try {
+ response.close();
+ httpClient.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ return html;
+
+ }
+
+
+ /**
+ * json参数方式POST提交
+ * @param url
+ * @param params
+ * @return
+ */
+ public static String doPost(String url, String params){
+ String strResult = "";
+ //设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(timeout * 1000)
+ .setTcpNoDelay(true).build();
+// AuthCache authCache = new BasicAuthCache();
+// authCache.put(proxy, new BasicScheme());
+// HttpClientContext localContext = HttpClientContext.create();
+// localContext.setAuthCache(authCache);
+ // 1. 获取默认的client实例
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ httpBuilder.setUserAgent(ua);
+ HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
+// HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
+// .setDefaultCredentialsProvider(credsProvider).build();
+ // 2. 创建httppost实例
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
+ HttpResponse resp = null;
+ try {
+ httpPost.setEntity(new StringEntity(params,"utf-8"));
+ resp = client.execute(httpPost);
+// resp = client.execute(httpPost,localContext);
+ StatusLine statusLine = resp.getStatusLine();
+ System.out.println("响应状态为:" + resp.getStatusLine());
+ int notFundCode = 404;
+ int successCode = 200;
+ if(statusLine.getStatusCode() == successCode){
+ // 7. 获取响应entity
+ HttpEntity respEntity = resp.getEntity();
+ strResult = EntityUtils.toString(respEntity, "UTF-8");
+ if(strResult.equals(Constants.EMPTY)){
+ strResult = "Download failed error is:reslut is null";
+ }
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+ return strResult;
+ }
+ public static String httpPost(String url,String params) {
+ String html="";
+ html = doPost(url,params);
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(5000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ html = doPost(url,params);
+ }
+ return html;
+ }
+ /**
+ * 绕过验证
+ *
+ * @return
+ * @throws NoSuchAlgorithmException
+ * @throws KeyManagementException
+ */
+ public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
+ SSLContext sc = SSLContext.getInstance("SSLv3");
+
+ // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
+ X509TrustManager trustManager = new X509TrustManager() {
+ @Override
+ public void checkClientTrusted(
+ java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
+ String paramString) throws CertificateException {
+ }
+
+ @Override
+ public void checkServerTrusted(
+ java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
+ String paramString) throws CertificateException {
+ }
+
+ @Override
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ };
+
+ sc.init(null, new TrustManager[] { trustManager }, null);
+ return sc;
+ }
+ /**
+ * 模拟请求
+ *
+ * @param url 资源地址
+ * @param map 参数列表
+ * @param encoding 编码
+ * @return
+ * @throws NoSuchAlgorithmException
+ * @throws KeyManagementException
+ * @throws IOException
+ * @throws ClientProtocolException
+ */
+ public static String httpsslGet(String url,Map ... headers) {
+ String html="";
+ CloseableHttpClient client = null;
+ HttpEntity responseEntity = null;
+ CloseableHttpResponse response = null;
+ try {
+ log.debug("DownLoadUtil------------->设置下载相关信息， start....");
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connManager.setMaxTotal(50);
+ connManager.setDefaultMaxPerRoute(10);
+ HttpClients.custom().setConnectionManager(connManager);
+ //设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ // 设置重定向策略
+ LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy();
+ //创建自定义的httpclient对象
+ client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build();
+// CloseableHttpClient client = HttpClients.createDefault();
+
+ HttpGet httpGet = new HttpGet(url);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
+ }
+ log.debug("DownLoadUtil------------->设置下载相关信息， end....");
+ try {
+ int notFundCode = 404;
+ int successCode = 200;
+ log.debug("DownLoadUtil------------->下载执行，start....");
+ httpGet.setConfig(config);
+ response = client.execute(httpGet);
+ log.debug("DownLoadUtil------------->下载执行，end....");
+ // 从响应模型中获取响应实体
+ StatusLine statusLine = response.getStatusLine();
+ log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is start ....");
+ System.out.println("响应状态为:" + response.getStatusLine());
+ responseEntity = response.getEntity();
+ log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is end ....");
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ System.out.println("响应内容长度为:" + responseEntity.getContentLength());
+ }
+ }else if(statusLine.getStatusCode() == notFundCode){
+ html = "页面404，正常结束请求即可
";
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }finally{
+ try {
+ responseEntity.getContent().close();
+ response.close();
+ client.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
+ return html;
+ }
+
+ public static String httpSSLGet(String url,Map ... headers) {
+ String html="";
+ html = httpsslGet(url,headers);
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(30000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ html = httpsslGet(url,headers);
+ }
+ return html;
+ }
+ public static String doPostFrom(String url,Map param,Map ... headers){
+ //设置超时时间
+ int timeout = 15;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+// AuthCache authCache = new BasicAuthCache();
+// authCache.put(proxy, new BasicScheme());
+// HttpClientContext localContext = HttpClientContext.create();
+// localContext.setAuthCache(authCache);
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ httpBuilder.setUserAgent(ua);
+// HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
+// .setDefaultCredentialsProvider(credsProvider).build();
+ HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpPost.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
+ httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
+ httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
+ httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36");
+// httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html");
+ }
+ // 创建请求参数
+ List list = new LinkedList<>();
+ for (String key : param.keySet()) {
+ BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
+ list.add(param1);
+ }
+ // 使用URL实体转换工具
+ String html="";
+ try {
+ UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
+ httpPost.setEntity(entityParam);
+ HttpResponse response = httpClient.execute(httpPost);
+// HttpResponse response = httpClient.execute(httpPost,localContext);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ int notFundCode = 404;
+ int successCode = 200;
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ }
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+
+ return html;
+
+ }
+ public static String httpPostForm(String url,Map params,Map ... headers) {
+ String html="";
+ html = doPostFrom(url,params);
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(5000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ html = doPostFrom(url,params,headers);
+ }
+ return html;
+ }
+
+ public static String dosslPost(String url,String params,Map ... headers) {
+ String html="";
+ CloseableHttpClient client = null;
+ HttpEntity responseEntity = null;
+ CloseableHttpResponse response = null;
+ try {
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ HttpClients.custom().setConnectionManager(connManager);
+ //设置超时时间
+ int timeout = 5;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ //创建自定义的httpclient对象
+ client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
+// CloseableHttpClient client = HttpClients.createDefault();
+ // 2. 创建httppost实例
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpPost.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
+ }
+
+ try {
+ httpPost.setEntity(new StringEntity(params,"utf-8"));
+ response = client.execute(httpPost);
+ int notFundCode = 404;
+ int successCode = 200;
+ // 从响应模型中获取响应实体
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ responseEntity = response.getEntity();
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ System.out.println("响应内容长度为:" + responseEntity.getContentLength());
+ }
+ }else if(statusLine.getStatusCode() == notFundCode){
+ html = "页面404，正常结束请求即可
";
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }finally{
+ try {
+ responseEntity.getContent().close();
+ response.close();
+ client.close();
+ } catch (UnsupportedOperationException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
+ return html;
+ }
+ public static String dosslPostForm(String url,Map param,Map ... headers) {
+ String html="";
+ try {
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ HttpClients.custom().setConnectionManager(connManager);
+ //设置超时时间
+ int timeout = 5;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ //创建自定义的httpclient对象
+ CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
+// CloseableHttpClient client = HttpClients.createDefault();
+ // 2. 创建httppost实例
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpPost.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
+ httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
+ }
+
+ // 创建请求参数
+ List list = new LinkedList<>();
+ for (String key : param.keySet()) {
+ BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
+ list.add(param1);
+ }
+ // 使用URL实体转换工具
+ try {
+ UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
+ httpPost.setEntity(entityParam);
+ HttpResponse response = client.execute(httpPost);
+// HttpResponse response = httpClient.execute(httpPost,localContext);
+ // 从响应模型中获取响应实体
+ int notFundCode = 404;
+ int successCode = 200;
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ }
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+
+
+ return html;
+ }
+ public static String httpSSLPostForm(String url,Map params,Map ...headers) {
+ String html="";
+ try {
+ html = dosslPostForm(url,params,headers);
+ } catch (Exception e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(30000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ try {
+ html = dosslPostForm(url,params,headers);
+ } catch (Exception e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ }
+ return html;
+ }
+ public static String httpSSLPost(String url,String params,Map ...headers) {
+ String html="";
+ try {
+ html = dosslPost(url,params,headers);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(30000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ try {
+ html = dosslPost(url,params,headers);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ }
+ return html;
+ }
+
+ /**
+ * 模拟客户端get请求
+ * @param url 模拟请求得url
+ * @param headers 头部信息，没有可以不传
+ * @return
+ */
+ public static String doGet(String url,Map ... headers){
+ //设置超时时间
+ int timeout = 15;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ httpBuilder.setUserAgent(ua);
+ HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
+ HttpGet httpGet = new HttpGet(url);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ }
+ String html="";
+ try {
+ int notFundCode = 404;
+ int successCode = 200;
+ HttpResponse response = httpClient.execute(httpGet);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ if(html.equals("")){
+ html = "Download failed error is:reslut is null";
+ }
+ }
+ }else if(statusLine.getStatusCode() == notFundCode){
+ html = "页面404，正常结束请求即可
";
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+ return html;
+
+ }
+
+ /**
+ * 文件下载
+ * @param fileURL
+ * @param saveDir
+ * @param fileName
+ * @throws IOException
+ */
+ public static void downloadFile(String fileURL, String saveFilePath) throws IOException {
+ CloseableHttpClient httpClient = HttpClients.createDefault();
+ HttpGet httpGet = new HttpGet(fileURL);
+
+ CloseableHttpResponse response = httpClient.execute(httpGet);
+
+ try {
+ int successCode = 200;
+ if (response.getStatusLine().getStatusCode() == successCode) {
+ // 设置文件保存路径
+
+ // 将响应实体写入文件
+ FileOutputStream outputStream = new FileOutputStream(saveFilePath);
+ response.getEntity().writeTo(outputStream);
+
+ // 关闭流
+ outputStream.close();
+ } else {
+ log.error("下载失败. HTTP 响应码: {}", response.getStatusLine().getStatusCode());
+ }
+ } finally {
+ response.close();
+ httpClient.close();
+ }
+ }
+
+ public static Map uploadFile(String url, String filePath) throws Exception {
+ File file = new File(filePath);
+
+ Map returnMap = new HashMap(32);
+ OkHttpClient client = getOkHttpClient();
+ // 设置文件上传的媒体类型
+ MediaType mediaType = MediaType.parse("application/octet-stream");
+ // 创建请求体，将文件添加到请求体中
+ RequestBody requestBody = RequestBody.create(mediaType, file);
+
+ // 创建多部分请求体，用于上传文件
+ MultipartBody multipartBody = new MultipartBody.Builder()
+ .setType(MultipartBody.FORM)
+ .addFormDataPart("file", file.getName(), requestBody)
+ .build();
+ // 创建上传文件的请求
+ Request request = new Request.Builder()
+ .url(url)
+ .post(multipartBody)
+ .build();
+
+ try (Response response = client.newCall(request).execute()) {
+ if (!response.isSuccessful()) {
+ throw new IOException("Failed to upload file: " + response);
+ }
+ String html = response.body().string();
+
+ try {
+ Map parse = (Map) JSON.parse(html);
+ Map data = (Map) parse.get("data");
+ String domain = (String) data.get("domain");
+ String src = (String) data.get("src");
+ String fileUrl = domain.concat(src);
+ returnMap.put("fileUrl", fileUrl);
+ } catch (Exception e) {
+ returnMap.put("fileUrl", html);
+ }
+ // 处理上传成功的响应
+ System.out.println("File uploaded successfully!");
+ }
+ return returnMap;
+ }
+ public static void main(String[] args) throws Exception {
+ String fileURL = "http://172.18.1.146:8080/group1/default/20240305/09/59/5/China’s Concern About Nuclear Wastewater May Be More About Politics Than Science.docx";
+ String saveFilePath = "D:\\工作使用\\analyze\\1010data\\政策\\China’s Concern About Nuclear Wastewater May Be More About Politics Than Science.docx";
+ String encodedUrl = URLEncoder.encode(fileURL, "utf-8");
+ System.out.println(encodedUrl);
+ downloadFile(encodedUrl, saveFilePath);
+ }
+}
diff --git a/src/main/java/com/bfd/utils/EncryptionUtil.java b/src/main/java/com/bfd/utils/EncryptionUtil.java
new file mode 100644
index 0000000..ead3f67
--- /dev/null
+++ b/src/main/java/com/bfd/utils/EncryptionUtil.java
@@ -0,0 +1,27 @@
+package com.bfd.utils;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * @author jian.mao
+ * @date 2023年3月10日
+ * @description
+ */
+public class EncryptionUtil {
+ public static String md5(String text) {
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update(text.getBytes());
+ byte[] bytes = md.digest();
+ StringBuilder sb = new StringBuilder();
+ for (byte b : bytes) {
+ sb.append(String.format("%02x", b & 0xff));
+ }
+ return sb.toString();
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/utils/FileUtil.java b/src/main/java/com/bfd/utils/FileUtil.java
new file mode 100644
index 0000000..fdc8dc1
--- /dev/null
+++ b/src/main/java/com/bfd/utils/FileUtil.java
@@ -0,0 +1,36 @@
+package com.bfd.utils;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * 文件工具类
+ * @author jian.mao
+ * @date 2023年7月14日
+ * @description
+ */
+public class FileUtil {
+
+ /**
+ * 数据写入文件
+ * @param Path 文件路径
+ * @param result 数据
+ * @throws IOException
+ */
+ public static void writeFile(String path,String result){
+ try {
+ FileWriter fw = new FileWriter(path,true);
+ fw.write(result+"\n");
+ fw.flush();
+ fw.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/utils/JsonUtil.java b/src/main/java/com/bfd/utils/JsonUtil.java
new file mode 100644
index 0000000..e4f20dd
--- /dev/null
+++ b/src/main/java/com/bfd/utils/JsonUtil.java
@@ -0,0 +1,32 @@
+package com.bfd.utils;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.entity.Constants;
+
+/**
+ * json工具
+ * @author jian.mao
+ * @date 2023年7月10日
+ * @description
+ */
+public class JsonUtil {
+
+ /**
+ * 校验字符串是list/map/str
+ * @param jsonString
+ * @return
+ */
+ public static String checkJsonType(String jsonString) {
+ try {
+ JSONObject.parseObject(jsonString);
+ return Constants.MAP_TYPE;
+ } catch (Exception e) {
+ try {
+ JSONObject.parseArray(jsonString);
+ return Constants.LIST_TYPE;
+ } catch (Exception ex) {
+ return Constants.STRING_TYPE;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/utils/OcrUtil.java b/src/main/java/com/bfd/utils/OcrUtil.java
new file mode 100644
index 0000000..7c4d28b
--- /dev/null
+++ b/src/main/java/com/bfd/utils/OcrUtil.java
@@ -0,0 +1,65 @@
+package com.bfd.utils;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import com.alibaba.fastjson.JSON;
+import com.bfd.entity.Constants;
+
+/**
+ * @author:jinming
+ * @className:ocrUtil
+ * @version:1.0
+ * @description:
+ * @Date:2023/8/1 16:38
+ */
+public class OcrUtil {
+ private static OkHttpClient okHttpClient;
+
+ private static OkHttpClient getOkHttpClient() {
+ if (okHttpClient == null) {
+ okHttpClient = new OkHttpClient();
+ }
+ return okHttpClient;
+ }
+
+ public static String doOcr(String url,String ocrApi) {
+ String text = "";
+ int reTryTimes = 3;
+ for (int i = 0; i < reTryTimes; i++) {
+ int okCode = 200;
+ OkHttpClient client = getOkHttpClient();
+ OkHttpClient.Builder builder = client.newBuilder().writeTimeout(600, TimeUnit.SECONDS).connectTimeout(600, TimeUnit.SECONDS).readTimeout(600, TimeUnit.SECONDS);
+ client = builder.build();
+ MediaType mediaType = MediaType.parse("application/json");
+ RequestBody body = RequestBody.create(mediaType, "{\"id\":\"\",\"url\":\"" + url + "\"}");
+ Request request = new Request.Builder()
+ .url(ocrApi)
+ .method("POST", body)
+ .addHeader("Content-Type", "application/json")
+ .build();
+ try {
+ Response response = client.newCall(request).execute();
+ String html = response.body().string();
+ Map dataMap = (Map) JSON.parse(html);
+ int code = (int) dataMap.get("code");
+ if (code == okCode) {
+ text = (String) dataMap.get("text");
+ }
+ if (text.equals(Constants.EMPTY)) {
+ break;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ return text;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/utils/OtherUtils.java b/src/main/java/com/bfd/utils/OtherUtils.java
new file mode 100644
index 0000000..fd8ae7c
--- /dev/null
+++ b/src/main/java/com/bfd/utils/OtherUtils.java
@@ -0,0 +1,33 @@
+package com.bfd.utils;
+
+import java.security.MessageDigest;
+
+/**
+ * 其他工具类
+ * @author jian.mao
+ * @date 2023年9月19日
+ * @description
+ */
+public class OtherUtils {
+
+
+
+ public static String getMd5(String string) {
+ try {
+ MessageDigest md5 = MessageDigest.getInstance("MD5");
+ byte[] bs = md5.digest(string.getBytes("UTF-8"));
+ StringBuilder sb = new StringBuilder(40);
+ for (byte x : bs) {
+ if ((x & 0xff) >> 4 == 0) {
+ sb.append("0").append(Integer.toHexString(x & 0xff));
+ } else {
+ sb.append(Integer.toHexString(x & 0xff));
+ }
+ }
+ return sb.toString();
+ } catch (Exception e) {
+
+ return "nceaform" + System.currentTimeMillis();
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/utils/QueueUtil.java b/src/main/java/com/bfd/utils/QueueUtil.java
new file mode 100644
index 0000000..18b3ac5
--- /dev/null
+++ b/src/main/java/com/bfd/utils/QueueUtil.java
@@ -0,0 +1,18 @@
+package com.bfd.utils;
+
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingDeque;
+
+/**
+ * @author:jinming
+ * @className:QueueUtil
+ * @version:1.0
+ * @description:
+ * @Date:2023/7/13 15:00
+ */
+public class QueueUtil {
+
+ public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>();
+
+ public static LinkedBlockingDeque sendQueue = new LinkedBlockingDeque();
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/utils/ThrowMessageUtil.java b/src/main/java/com/bfd/utils/ThrowMessageUtil.java
new file mode 100644
index 0000000..7697dfe
--- /dev/null
+++ b/src/main/java/com/bfd/utils/ThrowMessageUtil.java
@@ -0,0 +1,23 @@
+package com.bfd.utils;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+/**
+ * @author jian.mao
+ * @date 2023年3月22日
+ * @description
+ */
+public class ThrowMessageUtil {
+
+ /**
+ * 获取异常信息
+ * @param t
+ * @return
+ */
+ public static String getErrmessage(Throwable t){
+ StringWriter stringWriter=new StringWriter();
+ t.printStackTrace(new PrintWriter(stringWriter,true));
+ return stringWriter.getBuffer().toString();
+ }
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
new file mode 100644
index 0000000..913133d
--- /dev/null
+++ b/src/main/resources/application.yml
@@ -0,0 +1,92 @@
+logging:
+ level:
+ root: info
+ path: ../logs
+server:
+ port: 8018
+ servlet:
+ context-path: /IntelliModelManager
+ tomcat:
+ uri-encoding: utf-8
+ max-connections: 20000
+ max-http-form-post-size: 1
+ max-threads: 1000
+spring:
+ application:
+ name: 智能模型管理
+ datasource:
+ url: jdbc:mysql://172.18.1.147:3306/youzhi_cda_db?serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8&useSSL=true
+ username: root
+ password: baifendian123
+ driver-class-name: com.mysql.cj.jdbc.Driver
+ kafka:
+ bootstrap-servers: 172.18.1.146:9092,172.18.1.147:9092,172.18.1.148:9092
+ producer:
+ retries: 0
+ #当有多个消息需要被发送到同一个分区时，生产者会把它们放在同一个批次里。该参数指定了一个批次可以使用的内存大小，按照字节数计算。
+ batch-size: 16384
+ # 设置生产者内存缓冲区的大小。
+ buffer-memory: 33554432
+ # 键的序列化方式
+ key-serializer: org.apache.kafka.common.serialization.StringSerializer
+ # 值的序列化方式
+ value-serializer: org.apache.kafka.common.serialization.StringSerializer
+ # acks=0 ： 生产者在成功写入消息之前不会等待任何来自服务器的响应。
+ # acks=1 ： 只要集群的首领节点收到消息，生产者就会收到一个来自服务器成功响应。
+ # acks=all ：只有当所有参与复制的节点全部收到消息时，生产者才会收到一个来自服务器的成功响应。
+ acks: 1
+ consumer:
+ # 自动提交的时间间隔 在spring boot 2.X 版本中这里采用的是值的类型为Duration 需要符合特定的格式，如1S,1M,2H,5D
+ auto-commit-interval: 1S
+ # 该属性指定了消费者在读取一个没有偏移量的分区或者偏移量无效的情况下该作何处理：
+ # latest（默认值）在偏移量无效的情况下，消费者将从最新的记录开始读取数据（在消费者启动之后生成的记录）
+ # earliest ：在偏移量无效的情况下，消费者将从起始位置读取分区的记录
+ auto-offset-reset: earliest
+ # 是否自动提交偏移量，默认值是true,为了避免出现重复数据和数据丢失，可以把它设置为false,然后手动提交偏移量
+ enable-auto-commit: true
+ # 键的反序列化方式
+ key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
+ # 值的反序列化方式
+ value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
+ #消费组
+ group-id: test4
+ #消费者并发线程数
+ concurrency: 4
+ #超时时间
+ max-poll-interval-ms: 60000
+ #listener:
+ # 在侦听器容器中运行的线程数。
+ #concurrency: 5
+ #listner负责ack，每调用一次，就立即commit
+ #ack-mode: manual_immediate
+ #missing-topics-fatal: false
+ boot:
+ admin:
+ client:
+ url: http://172.18.1.147:8001
+ instance:
+ service-base-url: http://172.18.1.147:8018
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: "*"
+ endpoint:
+ health:
+ show-details: always
+ health:
+ elasticsearch:
+ enabled: false
+
+customize-kafka:
+ producer:
+ topic: produce_analyze
+task:
+ task-queue-path: ../data/taskQueue.txt
+ video-task-queue-path: ../data/videoTaskQueue.txt
+ thread-num: 1
+file:
+ path: ../files/
+gofast:
+ url: http://172.18.1.180:9980/group17/upload
\ No newline at end of file
diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml
new file mode 100644
index 0000000..abd8c3a
--- /dev/null
+++ b/src/main/resources/logback-spring.xml
@@ -0,0 +1,36 @@
+
+
+
+
+
+
+
+
+ true
+
+ ${logging.level}
+
+
+ ${logging.path}/IntelliModelManagerInfo.log
+
+
+ ${logging.path}/IntelliModelManagerInfo.log.%d{yyyy-MM-dd}
+ 7
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n
+ UTF-8
+
+
+
+
+
+
+
+
diff --git a/src/test/java/com/bfd/AppTest.java b/src/test/java/com/bfd/AppTest.java
new file mode 100644
index 0000000..7f816c2
--- /dev/null
+++ b/src/test/java/com/bfd/AppTest.java
@@ -0,0 +1,20 @@
+package com.bfd;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+/**
+ * Unit test for simple App.
+ */
+public class AppTest
+{
+ /**
+ * Rigorous Test :-)
+ */
+ @Test
+ public void shouldAnswerWithTrue()
+ {
+ assertTrue( true );
+ }
+}