From 8730171a006fc7117229269677a96c4f367b29c8 Mon Sep 17 00:00:00 2001 From: 55007 <55007@maojian> Date: Tue, 7 Jan 2025 18:13:55 +0800 Subject: [PATCH] =?UTF-8?q?mimax=E5=A4=A7=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .classpath | 40 + .gitignore | 2 + .project | 23 + .settings/org.eclipse.core.resources.prefs | 5 + .settings/org.eclipse.jdt.core.prefs | 9 + .settings/org.eclipse.m2e.core.prefs | 4 + README.md | 1 + pom.xml | 286 ++++++ src/main/java/com/bfd/qanda/Application.java | 24 + src/main/java/com/bfd/qanda/cache/ConfigCache.java | 37 + src/main/java/com/bfd/qanda/cache/LocalCache.java | 65 ++ .../com/bfd/qanda/controller/QandaController.java | 47 + src/main/java/com/bfd/qanda/entity/Constants.java | 168 +++ .../java/com/bfd/qanda/handler/MainHandler.java | 140 +++ .../com/bfd/qanda/process/PullVideoProcess.java | 174 ++++ .../java/com/bfd/qanda/process/QandaPorcess.java | 213 ++++ .../bfd/qanda/process/VideoGenerationProcess.java | 90 ++ .../java/com/bfd/qanda/service/QandaService.java | 24 + .../bfd/qanda/service/impl/QandaServiceImpl.java | 87 ++ src/main/java/com/bfd/qanda/utils/DataUtil.java | 48 + src/main/java/com/bfd/qanda/utils/DateUtil.java | 177 ++++ .../java/com/bfd/qanda/utils/DownLoadUtil.java | 1070 ++++++++++++++++++++ .../java/com/bfd/qanda/utils/EncryptionUtil.java | 27 + src/main/java/com/bfd/qanda/utils/FileUtil.java | 36 + .../com/bfd/qanda/utils/GPTResultParseUtil.java | 53 + src/main/java/com/bfd/qanda/utils/JsonUtil.java | 32 + src/main/java/com/bfd/qanda/utils/OtherUtils.java | 33 + src/main/java/com/bfd/qanda/utils/PauseTool.java | 92 ++ src/main/java/com/bfd/qanda/utils/QueueUtil.java | 18 + .../java/com/bfd/qanda/utils/SpringBootKafka.java | 46 + .../java/com/bfd/qanda/utils/ThrowMessageUtil.java | 23 + src/main/resources/application.yml | 102 ++ src/main/resources/logback-spring.xml | 36 + src/test/java/com/bfd/AppTest.java | 20 + 34 files changed, 3252 insertions(+) create mode 100644 .classpath create mode 100644 .gitignore create mode 100644 .project create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 .settings/org.eclipse.jdt.core.prefs create mode 100644 .settings/org.eclipse.m2e.core.prefs create mode 100644 README.md create mode 100644 pom.xml create mode 100644 src/main/java/com/bfd/qanda/Application.java create mode 100644 src/main/java/com/bfd/qanda/cache/ConfigCache.java create mode 100644 src/main/java/com/bfd/qanda/cache/LocalCache.java create mode 100644 src/main/java/com/bfd/qanda/controller/QandaController.java create mode 100644 src/main/java/com/bfd/qanda/entity/Constants.java create mode 100644 src/main/java/com/bfd/qanda/handler/MainHandler.java create mode 100644 src/main/java/com/bfd/qanda/process/PullVideoProcess.java create mode 100644 src/main/java/com/bfd/qanda/process/QandaPorcess.java create mode 100644 src/main/java/com/bfd/qanda/process/VideoGenerationProcess.java create mode 100644 src/main/java/com/bfd/qanda/service/QandaService.java create mode 100644 src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java create mode 100644 src/main/java/com/bfd/qanda/utils/DataUtil.java create mode 100644 src/main/java/com/bfd/qanda/utils/DateUtil.java create mode 100644 src/main/java/com/bfd/qanda/utils/DownLoadUtil.java create mode 100644 src/main/java/com/bfd/qanda/utils/EncryptionUtil.java create mode 100644 src/main/java/com/bfd/qanda/utils/FileUtil.java create mode 100644 src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java create mode 100644 src/main/java/com/bfd/qanda/utils/JsonUtil.java create mode 100644 src/main/java/com/bfd/qanda/utils/OtherUtils.java create mode 100644 src/main/java/com/bfd/qanda/utils/PauseTool.java create mode 100644 src/main/java/com/bfd/qanda/utils/QueueUtil.java create mode 100644 src/main/java/com/bfd/qanda/utils/SpringBootKafka.java create mode 100644 src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java create mode 100644 src/main/resources/application.yml create mode 100644 src/main/resources/logback-spring.xml create mode 100644 src/test/java/com/bfd/AppTest.java diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..1a0a8d2 --- /dev/null +++ b/.classpath @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..69f78ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/logs/ +/target/ \ No newline at end of file diff --git a/.project b/.project new file mode 100644 index 0000000..7e8119b --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + minimax_model + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..839d647 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..71df522 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,9 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.methodParameters=generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/README.md b/README.md new file mode 100644 index 0000000..c8554ed --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +接入minimax模型 diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..060321c --- /dev/null +++ b/pom.xml @@ -0,0 +1,286 @@ + + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.2.4.RELEASE + + com.bfd + minimax_model + 0.0.1-SNAPSHOT + + minimax_model + + http://www.example.com + + + UTF-8 + 1.8 + 1.8 + + + + + junit + junit + 4.11 + test + + + org.springframework.boot + spring-boot-starter-web + + + + de.codecentric + spring-boot-admin-starter-client + 2.2.4 + + + com.google.code.gson + gson + 2.8.8 + + + org.springframework.boot + spring-boot-test + + + + org.springframework + spring-test + 5.0.10.RELEASE + test + + + commons-io + commons-io + 2.11.0 + + + + com.alibaba + fastjson + 2.0.17 + + + + com.mchange + c3p0 + 0.9.5.5 + + + + com.squareup.okhttp3 + okhttp + 4.9.3 + + + org.apache.httpcomponents + httpclient + 4.5.3 + + + org.apache.httpcomponents + httpmime + 4.5.13 + + + commons-lang + commons-lang + 2.6 + + + + org.jetbrains.kotlin + kotlin-reflect + 1.6.21 + runtime + + + + org.jsoup + jsoup + 1.8.1 + + + org.apache.pdfbox + pdfbox + 2.0.28 + + + org.apache.poi + poi-scratchpad + 4.0.1 + + + org.apache.poi + poi + 4.0.1 + + + org.apache.poi + poi-ooxml + 4.0.1 + + + + org.apache.logging.log4j + log4j-api + 2.14.1 + + + org.apache.logging.log4j + log4j-core + 2.14.1 + + + + org.projectlombok + lombok + + + + + + + + org.springframework.kafka + spring-kafka + + + cn.hutool + hutool-all + 5.8.5 + + + junit + junit + + + + p6spy + p6spy + 3.9.0 + + + + commons-collections + commons-collections + 3.2.2 + + + + org.redisson + redisson-spring-boot-starter + 3.13.6 + + + org.springframework.boot + spring-boot-starter-data-redis + + + org.apache.curator + curator-framework + 5.2.0 + + + org.apache.curator + curator-recipes + 5.2.0 + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + org.springframework.boot + spring-boot-maven-plugin + + com.bfd.qanda.Application + ZIP + + + ${project.groupId} + ${project.artifactId} + + + + + + + repackage + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.1.1 + + + copy + package + + copy-dependencies + + + jar + jar + runtime + ${project.build.directory}/libs + + + + + + + + diff --git a/src/main/java/com/bfd/qanda/Application.java b/src/main/java/com/bfd/qanda/Application.java new file mode 100644 index 0000000..9c1d177 --- /dev/null +++ b/src/main/java/com/bfd/qanda/Application.java @@ -0,0 +1,24 @@ +package com.bfd.qanda; + + + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * 主入口 + * + * @author jian.mao + * @date 2023年7月4日 + * @description + */ +@SpringBootApplication +@EnableKafka +public class Application { + + public static void main(String[] args) { + SpringApplication.run(Application.class, args); + } +} \ No newline at end of file diff --git a/src/main/java/com/bfd/qanda/cache/ConfigCache.java b/src/main/java/com/bfd/qanda/cache/ConfigCache.java new file mode 100644 index 0000000..c18c4e0 --- /dev/null +++ b/src/main/java/com/bfd/qanda/cache/ConfigCache.java @@ -0,0 +1,37 @@ +package com.bfd.qanda.cache; + +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.concurrent.LinkedBlockingDeque; + +/** + * @author jian.mao + * @date 2022年11月11日 + * @description 静态变量类 + */ +@Slf4j +public class ConfigCache { + + /**启动条件**/ + public static boolean isStart = true; + /*****任务队列*****/ + public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> videoCommitTaskQueue = new LinkedBlockingDeque>(); + public static LinkedBlockingDeque> videoResultTaskQueue = new LinkedBlockingDeque>(); + + + /** + * 队列录入任务 + * @param queue + * @param task + */ + public static void putQueue(LinkedBlockingDeque> queue,Map task){ + //next app 写入队列准备调出 + try { + queue.put(task); + } catch (InterruptedException e) { + log.error("队列写入data失败---"); + } + } +} diff --git a/src/main/java/com/bfd/qanda/cache/LocalCache.java b/src/main/java/com/bfd/qanda/cache/LocalCache.java new file mode 100644 index 0000000..c692107 --- /dev/null +++ b/src/main/java/com/bfd/qanda/cache/LocalCache.java @@ -0,0 +1,65 @@ +package com.bfd.qanda.cache; + +import java.util.Calendar; +import java.util.Date; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * token 缓存 + * @author jian.mao + * @date 2023年11月8日 + * @description + */ +@Slf4j +public class LocalCache { + + private static final ConcurrentMap CACHE = new ConcurrentHashMap<>(64); + + private LocalCache() { + } + + public static LocalCache getInstance() { + return Inner.LOCAL_CACHE; + } + + public static String get(String key) { + Value v = LocalCache.CACHE.get(key); + if (v == null || new Date().after(v.end)) { + return ""; + } + + log.debug("get key:{},time left:{}s", key, (v.end.getTime() - new Date().getTime()) / 1000); + return v.value; + } + + public static void set(String key, String value, int expire, TimeUnit timeUnit) { + Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.SECOND, (int) timeUnit.toSeconds(expire)); + Value v = new Value(value, calendar.getTime()); + log.debug("put key:{}, expire time:{} ", key, calendar.getTime()); + LocalCache.CACHE.put(key, v); + } + + private static class Value { + + String value; + Date end; + + public Value(String value, Date time) { + this.value = value; + this.end = time; + } + } + + private static class Inner { + + private static final LocalCache LOCAL_CACHE = new LocalCache(); + } +} diff --git a/src/main/java/com/bfd/qanda/controller/QandaController.java b/src/main/java/com/bfd/qanda/controller/QandaController.java new file mode 100644 index 0000000..0674054 --- /dev/null +++ b/src/main/java/com/bfd/qanda/controller/QandaController.java @@ -0,0 +1,47 @@ +package com.bfd.qanda.controller; + +import javax.annotation.Resource; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; + +import com.bfd.qanda.service.QandaService; + + +/** + * @author jian.mao + * @date 2023年11月8日 + * @description + */ +@Controller +@RequestMapping("/qanda") +@Slf4j +public class QandaController { + + @Resource + private QandaService qandaService; + @PostMapping("/putQuestion") + @ResponseBody + public String putQuestion(@RequestBody String dataJson){ + String response = qandaService.putQuestion(dataJson); + return response; + } + @PostMapping("/videoGeneration") + @ResponseBody + public String videoGeneration(@RequestBody String dataJson){ + String response = qandaService.videoGeneration(dataJson); + return response; + } + + @RequestMapping(value = "/hello", method = RequestMethod.GET) + @ResponseBody + public String hello(String param, String token) { + return "123"; + } +} diff --git a/src/main/java/com/bfd/qanda/entity/Constants.java b/src/main/java/com/bfd/qanda/entity/Constants.java new file mode 100644 index 0000000..c9d132c --- /dev/null +++ b/src/main/java/com/bfd/qanda/entity/Constants.java @@ -0,0 +1,168 @@ +package com.bfd.qanda.entity; + + +/** + * 常量实体类 + * @author jian.mao + * @date 2022年11月15日 + * @description + */ +public class Constants { + + /*************************蓝图常量key名称*********************************/ + public final static String SCHEDULING = "scheduling"; + public final static String TYPE = "type"; + public final static String INTERVAL = "interval"; + public final static String CREATED = "created"; + public final static String LAST_EDIT = "last_edit"; + public final static String BLUEPRINT_ID = "blueprint_id"; + public final static String BLUEPRINTID = "blueprintId"; + public final static String BLUEPRINT_NAME = "name"; + public final static String SCENARIO = "scenario"; + public final static String AUTOCOMMITTRIGGERLAST = "autoCommitTriggerLast"; + public final static String FRESHVARIABLES = "freshVariables"; + public final static String AUTOCOMMIT = "autoCommit"; + public final static String MAXERRORS = "maxErrors"; + public final static String DATALOSS = "dataloss"; + public final static String POSITION = "position"; + public final static String SCENES_ID = "scenes_id"; + public final static String SCENESID = "scenesId"; + public final static String MULTI_BRANCH = "multi_branch"; + + public final static String SINGLE = "single"; + /**已重试次数**/ + public final static String ERROR_TIME = "error_time"; + public final static String PREVIOUS_RESULT = "previous_result"; + + /****数据id*****/ + public final static String BUSINESSKEY = "businessKey"; + + + /*************************metadata常量key名称*********************************/ + public final static String LABEL_COL = "label_col"; + public final static String LABEL = "label"; + public final static String USER = "user"; + public final static String ADMIN = "admin"; + public final static String ADDRESS = "address"; + public final static String DATASOURCE = "datasource"; + public final static String INDEX = "index"; + + /*************************app常量key名称*********************************/ + public final static String APPS = "apps"; + public final static String TRANSFER_ID = "transfer_id"; + public final static String MODULE = "module"; + public final static String VERSION = "version"; + public final static String METADATA = "metadata"; + public final static String APP_NAME = "name"; + public final static String DESCRIBE = "describe"; + public final static String NEXT_APP_ID = "next_app_id"; + public final static String EDGE_ID = "edge_id"; + public final static String START_ID = "start_id"; + public final static String END_ID = "end_id"; + + public final static String WAIT_CONDITION = "wait_condition"; + public final static String START_TAG = "start_tag"; + + /*************************module类型*********************************/ + public final static String FILE = "file"; + public final static String OCR = "OCR"; + public final static String FILTER = "Filter"; + public final static String CHATGPT = "ChatGPT"; + public final static String MYSQL = "mysql"; + + /*************************other类型*********************************/ + public final static String UNDERLINE = "_"; + public final static String RESULT_TOPIC = null; + public static final String EMPTY = ""; + public static final String HTTP = "http"; + public static final String REQUEST_ERROR_MESSAGE = "Download failed error is"; + public static final String REQUEST_RESULT = "result"; + public static final String REQUEST_RESULT_RESULTS = "results"; + public static final String MAP_TYPE = "Map"; + public static final String LIST_TYPE = "List"; + public static final String STRING_TYPE = "String"; + public static final String DOCUMENT_TYPE = "doc"; + public static final String FILTER_ZH = "过滤器"; + + public static final String JSON_SELE_SYMBOL = "$."; + public static final String LEFT_BRACKETS = "["; + public static final String RIGTH_BRACKETS = "]"; + public static final String TASKTYPE = "taskType"; + public static final Integer USER_TYPE = 1; + public static final Integer KEYWORD_TYPE = 0; + public static final Integer DETAIL_TYPE = 2; + public static final String CID = "cid"; + public static final String SITETYPE = "siteType"; + public static final Integer DEFULT_SUBJECTID = 304864; + public static final Integer DEFULT_CRAWLCYCLICITYTIME = 1440; + public static final String CRAWLENDTIME = "crawlEndTime"; + public static final String CRAWLSTARTTIME = "crawlStartTime"; + public static final String CRAWLPAGETYPES = "crawlPageTypes"; + public static final String APPID = "113ic"; + public static final String APP_ID = "appId"; + public final static String ID = "id"; + public static final Integer DEFULT_CRAWLPERIODHOUR = 24; + public static final String CREATEUSERID = "662015832180933762"; + public static final String CRAWL_ADD_URL = "https://caiji.percent.cn/api/crawl/remote/task/save"; + public static final String CRAWLKEYWORD = "crawlKeyword"; + public static final String ATTACHTAG = "attachTag"; + public static final String ATTACHTAG_VALUE = "analyze"; + public static final String KEYWORD = "keyword"; + public static final String SITEID = "siteId"; + public static final String RESULTS = "results"; + public static final String RESULT = "result"; + public static final String CRAWLDATAFLAG = "crawlDataFlag"; + public static final String CRAWLDATAFLAG_PREFIX = "\"crawlDataFlag\":\"keyword:"; + public static final String TID = "tid"; + public static final Long TIME_OUT = 1800000L; + public static final String ATTR = "attr"; + public static final String HASVIDEO = "hasVideo"; + public static final String CRAWL_END_MARK = "crawl_end_mark"; + public static final String CRAWL_END_MESSAGE = "crawl_end_message"; + public static final String CRAWL_END_MESSAGE_VALUE = "数据采集完成"; + public static final String SUBJECTID = "subjectId"; + public static final String TASKID = "taskId"; + public static final String TASK_ID = "task_id"; + public static final int SUCCESS_CODE = 200; + public static final String WEB_URL_SUFFIX = "/api/aogeo/api/cda/caiji/status"; + public static final String STATUS = "status"; + /************************redis*************************************/ + public static final String LOCK_KEY = "myLock"; + public static final long LOCK_EXPIRE_TIME = 300000; + + /************************应用参数*************************************/ + public static final String CODE = "code"; + public static final String MESSAGES = "messages"; + public static final String INPUT = "input"; + public static final String OUTPUT = "output"; + public static final String FORM = "form"; + public static final String FIELD = "field"; + public static final String VALUE = "value"; + public static final String DATA = "data"; + public static final String COLON_EN = ":"; + /******************************admin*******************************/ + public static final String AUTHORIZATION = "authorization"; + public static final String TEMPERATURE = "temperature"; + public static final String FIELDTYPE = "fieldType"; + public static final String TOP_P = "top_p"; + public static final String MODEL = "model"; + public static final String PROMPT = "prompt"; + public static final String MINIMAX_TURBO_URL = "https://api.minimax.chat/v1/text/chatcompletion_v2"; + public static final String MINIMAX_VIDEO_COMMIT_URL = "https://api.minimax.chat/v1/video_generation"; + public static final String MINIMAX_VIDEO_QUERY_URL = "https://api.minimax.chat/v1/query/video_generation?task_id="; + public static final String MINIMAX_VIDEO_PULL_URL = "https://api.minimax.chat/v1/files/retrieve?file_id="; + public static final String CHOICES = "choices"; + public static final String CONTENT = "content"; + public static final String MESSAGE = "message"; + public static final String MASK_SENSITIVE_INFO = "mask_sensitive_info"; + public static final String MAX_TOKENS = "max_tokens"; + public static final int CHAT_TYPE_ONE = 1; + public static final String COMMITRESULT = "commitResult"; + public static final String BASE_RESP = "base_resp"; + public static final String STATUS_CODE = "status_code"; + public static final String FILE_ID = "file_id"; + + public static final String DOMAIN_NAME = "https://zyzs.pontoaplus.com"; + public static final String CLUSTER_ADDRESS = "http://172.18.1.180:9980"; + public static final String TRACE = "trace"; +} diff --git a/src/main/java/com/bfd/qanda/handler/MainHandler.java b/src/main/java/com/bfd/qanda/handler/MainHandler.java new file mode 100644 index 0000000..04065ae --- /dev/null +++ b/src/main/java/com/bfd/qanda/handler/MainHandler.java @@ -0,0 +1,140 @@ +package com.bfd.qanda.handler; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingDeque; + +import javax.annotation.Resource; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.core.annotation.Order; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.qanda.cache.ConfigCache; +import com.bfd.qanda.process.PullVideoProcess; +import com.bfd.qanda.process.QandaPorcess; +import com.bfd.qanda.process.VideoGenerationProcess; +import com.bfd.qanda.utils.FileUtil; +import com.bfd.qanda.utils.PauseTool; + + +/** + * 启动处理入口 + * + * @author jian.mao + * @date 2023年11月3日 + * @description + */ + +@Component +@Order(value = 1) +@Slf4j +public class MainHandler implements ApplicationRunner { + + @Value("${task.task-queue-path}") + private String taskPath; + @Value("${task.thread-num}") + private Integer threadNum; + @Resource + QandaPorcess qandaPorcess; + @Resource + PullVideoProcess pullVideoProcess; + @Resource + VideoGenerationProcess videoGenerationProcess; + @Resource + private StringRedisTemplate stringRedisTemplate; + @Value("${zookeeper.connection-string}") + private String connectionString; + @Value("${zookeeper.publish-node}") + private String nodePath; + + @Override + public void run(ApplicationArguments args) throws Exception { + PauseTool pauseTool = new PauseTool(); + pauseTool.initializeRedisCache(stringRedisTemplate); + pauseTool.setupZookeeperListener(connectionString, nodePath); + for (Integer i = 0; i < threadNum; i++) { + //开启问答处理线程 + new Thread(qandaPorcess).start(); + log.info("开启问答处理线程-----"); + //开启问答处理线程 + new Thread(pullVideoProcess).start(); + log.info("开启视频生成提交线程-----"); + //开启问答处理线程 + new Thread(videoGenerationProcess).start(); + log.info("开启视频拉取线程-----"); + } + + //停止处理 + waitDown(); + //启动加载缓存任务 + readTask(taskPath, ConfigCache.taskQueue); + } + + + @SuppressWarnings("unchecked") + public static void readTask(String path, LinkedBlockingDeque> queue) { + File file = new File(path); + if (file.exists()) { + List tasks = null; + try { + tasks = FileUtils.readLines(file, "UTF-8"); + } catch (IOException e) { + e.printStackTrace(); + } + for (String taskStr : tasks) { + Map task = JSONObject.parseObject(taskStr); + try { + queue.put(task); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + file.delete(); + } + } + + /** + * 结束触发钩子 + */ + public void waitDown() { + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + // 停止线程 + ConfigCache.isStart = false; + log.info("stop-------"); + writeTsskToFile(); + } + }); + } + + + /** + * 任务持久化到硬盘 + */ + public void writeTsskToFile() { + while (true) { + if (ConfigCache.taskQueue.size() > 0) { + try { + Map task = ConfigCache.taskQueue.take(); + FileUtil.writeFile(taskPath, JSONObject.toJSONString(task)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } else { + log.info("taskQueue write is file end"); + break; + } + } + } +} diff --git a/src/main/java/com/bfd/qanda/process/PullVideoProcess.java b/src/main/java/com/bfd/qanda/process/PullVideoProcess.java new file mode 100644 index 0000000..ed0654b --- /dev/null +++ b/src/main/java/com/bfd/qanda/process/PullVideoProcess.java @@ -0,0 +1,174 @@ +package com.bfd.qanda.process; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.qanda.cache.ConfigCache; +import com.bfd.qanda.entity.Constants; +import com.bfd.qanda.utils.DateUtil; +import com.bfd.qanda.utils.DownLoadUtil; +import com.bfd.qanda.utils.PauseTool; +import com.bfd.qanda.utils.SpringBootKafka; + +/** + * 视频拉取线程 + * @author jian.mao + * @date 2024年9月14日 + * @description + */ +@Component +@Slf4j +public class PullVideoProcess implements Runnable{ + @Autowired + private SpringBootKafka springBootKafka; + @Value("${customize-kafka.producer.topic}") + private String topic; + + @Value("${file.path}") + private String filePath; + + @Value("${gofast.url}") + private String uploadUrl; + + private final String QUEUEING = "Queueing"; + private final String PROCESSING = "Processing"; + private final String SUCCESS = "Success"; + private final String FAIL = "Fail"; + @Override + public void run() { + // TODO Auto-generated method stub + while (ConfigCache.isStart) { + Map task = null; + try { + task = ConfigCache.videoResultTaskQueue.take(); + log.info("视频结果获取任务队列长度：{}",ConfigCache.videoResultTaskQueue.size()); + log.info("视频结果获取任务：{}",JSONObject.toJSONString(task)); + //输入配置 + Map input = (Map) task.get(Constants.INPUT); + int scenesId = (int) task.get(Constants.SCENES_ID); + int version = (int) task.get(Constants.VERSION); + String pauseKey = scenesId + "_" + version; + if (!PauseTool.CACHE.containsKey(pauseKey)) { + log.info("流程:{}的版本:{}已失效,任务跳过", scenesId, version); + continue; + } + //提交视频 + log.info("-----------------拉取生成视频-----------------"); + //请求身份校验 api-key + String authorization = "Bearer " +(String) input.get(Constants.AUTHORIZATION); + Map commitReulst = JSONObject.parseObject((String)task.get(Constants.COMMITRESULT)); + Map baseResp = (Map) commitReulst.get(Constants.BASE_RESP); + int statusCode = (int) baseResp.get(Constants.STATUS_CODE); + Map result = new HashMap(16); + String id = UUID.randomUUID().toString(); + if(statusCode == 0){ + //提交成功，获取视频 + String taskId = (String) commitReulst.get(Constants.TASK_ID); + String gofastUrl = pullVideo(authorization,taskId); + if(gofastUrl.equals(QUEUEING) || gofastUrl.equals(PROCESSING)){ + //视频还未生成- 放回队列 + ConfigCache.videoResultTaskQueue.put(task); + }else if(gofastUrl.equals(FAIL)){ + //视频生成失败 + Map results = new HashMap(16); + //遍历入库返回结果，拼接响应内容 + results.put("isLast", 1); + results.put("id", id); + result.put(Constants.RESULTS, JSONObject.toJSONString(results)); + result.put(Constants.MESSAGE, "视频生成失败"); + result.put(Constants.STATUS, 2); + task.put(Constants.RESULT, result); + //发送kafka + springBootKafka.send(topic, JSONObject.toJSONString(task)); + log.info("数据流转至下游-------"); + }else{ + //结果集 + Map results = new HashMap(16); + //遍历入库返回结果，拼接响应内容 + results.put("isLast", 1); + results.put("id", id); + results.put("url", gofastUrl); + result.put(Constants.RESULTS, JSONObject.toJSONString(results)); + result.put(Constants.MESSAGE, "成功"); + result.put(Constants.STATUS, 1); + task.put(Constants.RESULT, result); + //发送kafka + springBootKafka.send(topic, JSONObject.toJSONString(task)); + log.info("数据流转至下游-------"); + } + }else{ + //提交失败---直接返回 + //结果集 + Map results = new HashMap(16); + //遍历入库返回结果，拼接响应内容 + results.put("isLast", 1); + results.put("id", id); + result.put(Constants.RESULTS, JSONObject.toJSONString(results)); + result.put(Constants.MESSAGE, baseResp.get("status_msg")); + result.put(Constants.STATUS, 2); + task.put(Constants.RESULT, result); + //发送kafka + springBootKafka.send(topic, JSONObject.toJSONString(task)); + log.info("数据流转至下游-------"); + } + //10s 拉取一次结果 + DateUtil.sleep(1000*10); + } catch (Throwable e) { + // TODO: handle exception + log.error("视频获取结果异常：{}",e); + } + } + } + /** + * 视频获取 + * @param authorization + * @param taskId + * @return + */ + private String pullVideo(String authorization, String taskId) { + String videoUrl = Constants.MINIMAX_VIDEO_QUERY_URL + taskId; + Map header = new HashMap(16); + header.put(Constants.AUTHORIZATION, authorization); + String queryRs = DownLoadUtil.doGet(videoUrl, header); + JSONObject queryJson = JSONObject.parseObject(queryRs); + String status = queryJson.getString(Constants.STATUS); + if(status.equals(QUEUEING)){ + log.info("{}:...队列中...",taskId); + return QUEUEING; + }else if(status.equals(PROCESSING)){ + log.info("{}:...生成中...",taskId); + return PROCESSING; + }else if(status.equals(SUCCESS)){ + log.info("{}:视频生成成功:{}",taskId,queryRs); + //获取视频信息 + String generationUrl = Constants.MINIMAX_VIDEO_PULL_URL + queryJson.get(Constants.FILE_ID); + String videoRes = DownLoadUtil.doGet(generationUrl, header); + log.info("{}:视频信息:{}",taskId,videoRes); + String fileName = UUID.randomUUID().toString() + ".mp4"; + String saveFilePath = filePath + fileName; + //获取视频链接 + JSONObject video = JSONObject.parseObject(videoRes); + Map file = (Map) video.get("file"); + String videoDownloadUrl = (String) file.get("download_url"); + //下载 + DownLoadUtil.downloadFile(videoDownloadUrl, saveFilePath); + //上传 +域名替换 + String gofastUrl = DownLoadUtil.uploadFile(uploadUrl, saveFilePath).replace(Constants.CLUSTER_ADDRESS,Constants.DOMAIN_NAME); + //删除 + DownLoadUtil.delFile(saveFilePath); + return gofastUrl; + }else { + log.info("{}:视频生成失败:{}",taskId,queryRs); + return FAIL; + } + } + +} diff --git a/src/main/java/com/bfd/qanda/process/QandaPorcess.java b/src/main/java/com/bfd/qanda/process/QandaPorcess.java new file mode 100644 index 0000000..27c8840 --- /dev/null +++ b/src/main/java/com/bfd/qanda/process/QandaPorcess.java @@ -0,0 +1,213 @@ +package com.bfd.qanda.process; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import com.bfd.qanda.utils.GPTResultParseUtil; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + + + +import com.alibaba.fastjson.JSONObject; +import com.bfd.qanda.cache.ConfigCache; +import com.bfd.qanda.entity.Constants; +import com.bfd.qanda.utils.DataUtil; +import com.bfd.qanda.utils.DownLoadUtil; +import com.bfd.qanda.utils.PauseTool; +import com.bfd.qanda.utils.SpringBootKafka; + +/** + * @author jian.mao + * @date 2023年11月8日 + * @description + */ +@Component +@Slf4j +public class QandaPorcess implements Runnable { + @Autowired + private SpringBootKafka springBootKafka; + @Value("${customize-kafka.producer.topic}") + private String topic; + + @Override + public void run() { + while (ConfigCache.isStart) { + Map task = null; + try { + task = ConfigCache.taskQueue.take(); + log.info("任务队列长度：{}",ConfigCache.taskQueue.size()); + log.info("任务：{}",JSONObject.toJSONString(task)); + //结果接收 + Map result = new HashMap(16); + //结果内容 + StringBuffer chatContent = new StringBuffer(); + //输入配置 + Map input = (Map) task.get(Constants.INPUT); + //输出配置 + Map output = (Map) task.get(Constants.OUTPUT); + //数据源 + Map data = (Map) task.get(Constants.DATA); + int scenesId = (int) task.get(Constants.SCENES_ID); + int version = (int) task.get(Constants.VERSION); + String pauseKey = scenesId + "_" + version; + if (!PauseTool.CACHE.containsKey(pauseKey)) { + log.info("流程:{}的版本:{}已失效,任务跳过", scenesId, version); + continue; + } + //请求身份校验 api-key + String authorization = "Bearer " +(String) input.get(Constants.AUTHORIZATION); + //fieldType：自定义输出字段： 0 关闭，1-开启，如果开启则拼接form到output里（如果关闭，则取默认的output拼接） + int fieldType = (int) input.get(Constants.FIELDTYPE); + Float temperature = Float.valueOf(input.get(Constants.TEMPERATURE).toString()); + Float topP = Float.valueOf(input.get(Constants.TOP_P).toString()); + String model = (String) input.get(Constants.MODEL).toString(); + List> prompt = (List>) input.get(Constants.PROMPT); + String answerStr = qandaRequest(authorization, model, temperature, topP, prompt, data); + log.info("answerStr:" + answerStr); + Map answer = JSONObject.parseObject(answerStr); + try { + //请求成功，正常解析 + List> choices = (List>) answer.get(Constants.CHOICES); + Map message = (Map) choices.get(0).get(Constants.MESSAGE); + chatContent.append(message.get(Constants.CONTENT)); + } catch (Exception e) { + log.error("问答接口响应体异常:{}", answerStr, e); + // TODO: handle exception + //结果集 + Map results = new HashMap(16); + //遍历入库返回结果，拼接响应内容 + results.put("isLast", 1); + results.put("content", answerStr); + result.put(Constants.RESULTS, JSONObject.toJSONString(results)); + result.put(Constants.MESSAGE, "问答异常"); + result.put(Constants.STATUS, 2); + task.put(Constants.RESULT, result); + //发送kafka + springBootKafka.send(topic, JSONObject.toJSONString(task)); + log.info("数据流转至下游-------"); + continue; + } + Map results = new HashMap(16); + results.put(Constants.ID, UUID.randomUUID().toString()); + results.put(Constants.CONTENT, chatContent.toString()); + if (fieldType != 0) { + results.remove(Constants.CONTENT); + try { + //请求成功，正常解析 + List> choices = (List>) answer.get(Constants.CHOICES); + Map message = (Map) choices.get(0).get(Constants.MESSAGE); + String reponseContent = (String) message.get(Constants.CONTENT); + Map stringObjectMap = GPTResultParseUtil.parseGPTResult(output, reponseContent); + results.putAll(stringObjectMap); + } catch (Exception e) { + log.error("问答接口响应体异常:{}", answerStr, e); + // TODO: handle exception + //结果集 + //遍历入库返回结果，拼接响应内容 + results.put("isLast", 1); + results.put("content", answerStr); + result.put(Constants.RESULTS, JSONObject.toJSONString(results)); + result.put(Constants.MESSAGE, "问答异常"); + result.put(Constants.STATUS, 2); + task.put(Constants.RESULT, result); + //发送kafka + springBootKafka.send(topic, JSONObject.toJSONString(task)); + log.info("数据流转至下游-------"); + continue; + } + } + results.put("isLast", 1); + result.put(Constants.RESULTS, JSONObject.toJSONString(results)); + result.put(Constants.MESSAGE, "成功"); + result.put(Constants.STATUS, 1); + task.put(Constants.RESULT, result); + //发送kafka + springBootKafka.send(topic, JSONObject.toJSONString(task)); + log.info("数据流转至下游-------"); + } catch (Throwable e) { + log.error("问答处理异常，", e); + //结果集 + Map results = new HashMap(16); + Map result = new HashMap(16); + //遍历入库返回结果，拼接响应内容 + results.put("isLast", 1); + results.put("content", e.getMessage()); + result.put(Constants.RESULTS, JSONObject.toJSONString(results)); + result.put(Constants.MESSAGE, "异常"); + result.put(Constants.STATUS, 2); + task.put(Constants.RESULT, result); + //发送kafka + springBootKafka.send(topic, JSONObject.toJSONString(task)); + log.info("数据流转至下游-------"); + + + } + + } + } + + /** + * 问题请求 + * + * @param authorization + * @param temperature + * @param topP + * @param prompt + * @return + */ + private String qandaRequest(String authorization, String model, Float temperature, Float topP, List> prompt, Map data) { + //新建聊天话术 + StringBuffer chatContent = new StringBuffer(); + for (Map map : prompt) { + if (Integer.valueOf(map.get(Constants.TYPE).toString()) == Constants.CHAT_TYPE_ONE) { + chatContent.append(map.get(Constants.VALUE)); + } else { + String jsonPath = (String) map.get(Constants.VALUE); + chatContent.append(DataUtil.getValue(jsonPath, data).toString()); + } + } + Map headers = new HashMap(16); + Map params = new HashMap(16); + List> prompts = buildParam(chatContent.toString(), data); + + params.put(Constants.MODEL, model); + params.put(Constants.MESSAGES, prompts); + params.put(Constants.TEMPERATURE, temperature); + params.put(Constants.TOP_P, topP); + params.put(Constants.MASK_SENSITIVE_INFO, false); + params.put(Constants.MAX_TOKENS, 245760); + + headers.put(Constants.AUTHORIZATION, authorization); + headers.put("Content-Type", "application/json"); + String html = DownLoadUtil.doPost(Constants.MINIMAX_TURBO_URL, JSONObject.toJSONString(params), headers); + return html; + } + + /** + * 参数构建 + * + * @param prompt + * @param data + * @return + */ + private List> buildParam(String prompt, Map data) { + //聊天体 + List> prompts = new ArrayList>(); + log.info("question---{}", prompt); + Map chat1 = new HashMap(16); + chat1.put("role", "user"); + chat1.put("content", prompt); + prompts.add(chat1); + return prompts; + } + + +} diff --git a/src/main/java/com/bfd/qanda/process/VideoGenerationProcess.java b/src/main/java/com/bfd/qanda/process/VideoGenerationProcess.java new file mode 100644 index 0000000..1b0db22 --- /dev/null +++ b/src/main/java/com/bfd/qanda/process/VideoGenerationProcess.java @@ -0,0 +1,90 @@ +package com.bfd.qanda.process; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.qanda.cache.ConfigCache; +import com.bfd.qanda.entity.Constants; +import com.bfd.qanda.utils.DataUtil; +import com.bfd.qanda.utils.DownLoadUtil; +import com.bfd.qanda.utils.PauseTool; +import com.bfd.qanda.utils.SpringBootKafka; + +/** + * 视频生成处理线程 + * @author jian.mao + * @date 2024年9月14日 + * @description + */ +@Component +@Slf4j +public class VideoGenerationProcess implements Runnable{ + + @Override + public void run() { + // TODO Auto-generated method stub + while (ConfigCache.isStart) { + Map task = null; + try { + task = ConfigCache.videoCommitTaskQueue.take(); + log.info("视频生成提交任务队列长度：{}",ConfigCache.videoCommitTaskQueue.size()); + log.info("频生成提交任务：{}",JSONObject.toJSONString(task)); + //输入配置 + Map input = (Map) task.get(Constants.INPUT); + //输出配置 + Map output = (Map) task.get(Constants.OUTPUT); + //数据源 + Map data = (Map) task.get(Constants.DATA); + int scenesId = (int) task.get(Constants.SCENES_ID); + int version = (int) task.get(Constants.VERSION); + String pauseKey = scenesId + "_" + version; + if (!PauseTool.CACHE.containsKey(pauseKey)) { + log.info("流程:{}的版本:{}已失效,任务跳过", scenesId, version); + continue; + } + //提交视频 + log.info("-----------------提交视频生成任务-----------------"); + //请求身份校验 api-key + String authorization = "Bearer " +(String) input.get(Constants.AUTHORIZATION); + String model = (String) input.get(Constants.MODEL).toString(); + List> prompt = (List>) input.get(Constants.PROMPT); + String commitResult = commit(authorization, model, prompt, data); + log.info("视频生成提交结果：{}",commitResult); + //放入视频获取队列 + task.put(Constants.COMMITRESULT, commitResult); + ConfigCache.videoResultTaskQueue.put(task); + }catch(Throwable e){ + log.error("视频提交异常：{}",e); + } + } + } + + private String commit(String authorization, String model, List> prompt, Map data) { + //新建聊天话术 + StringBuffer chatContent = new StringBuffer(); + for (Map map : prompt) { + if (Integer.valueOf(map.get(Constants.TYPE).toString()) == Constants.CHAT_TYPE_ONE) { + chatContent.append(map.get(Constants.VALUE)); + } else { + String jsonPath = (String) map.get(Constants.VALUE); + chatContent.append(DataUtil.getValue(jsonPath, data).toString()); + } + } + Map headers = new HashMap(16); + Map params = new HashMap(16); + params.put(Constants.MODEL, model); + params.put(Constants.PROMPT, chatContent.toString()); + headers.put(Constants.AUTHORIZATION, authorization); + headers.put("Content-Type", "application/json"); + String html = DownLoadUtil.doPost(Constants.MINIMAX_VIDEO_COMMIT_URL, JSONObject.toJSONString(params), headers); + return html; + } +} diff --git a/src/main/java/com/bfd/qanda/service/QandaService.java b/src/main/java/com/bfd/qanda/service/QandaService.java new file mode 100644 index 0000000..f20af54 --- /dev/null +++ b/src/main/java/com/bfd/qanda/service/QandaService.java @@ -0,0 +1,24 @@ +package com.bfd.qanda.service; + +/** + * 问答业务层 + * @author jian.mao + * @date 2023年11月8日 + * @description + */ +public interface QandaService { + + /** + * 问答 + * @param dataJson 接口参数 + * @return + */ + public String putQuestion(String dataJson); + + /** + * 视频生成 + * @param dataJson 接口参数 + * @return + */ + public String videoGeneration(String dataJson); +} diff --git a/src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java b/src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java new file mode 100644 index 0000000..c9f84a5 --- /dev/null +++ b/src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java @@ -0,0 +1,87 @@ +package com.bfd.qanda.service.impl; + +import java.util.HashMap; +import java.util.Map; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Service; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.qanda.cache.ConfigCache; +import com.bfd.qanda.entity.Constants; +import com.bfd.qanda.service.QandaService; + +/** + * 业务层实现类 + * + * @author jian.mao + * @date 2023年11月8日 + * @description + */ +@Service +@Slf4j +public class QandaServiceImpl implements QandaService { + + @Override + public String putQuestion(String dataJson) { + Map response = new HashMap<>(16); + int code = 200; + String message = "success"; + Map task = null; + try { + task = JSONObject.parseObject(dataJson); + } catch (Exception e) { + log.error("参数结构不合法，", e); + code = 100010; + message = "参数不合法"; + } + // 写入队列 + try { + if(task.containsKey(Constants.TRACE) && (boolean)task.get(Constants.TRACE)){ + ConfigCache.taskQueue.putFirst(task); + }else{ + ConfigCache.taskQueue.put(task); + } + } catch (InterruptedException e) { + log.error("任务写入等待队列异常，", e); + code = 100011; + message = "任务写入等待队列失败"; + } + response.put(Constants.CODE, code); + response.put(Constants.MESSAGE, message); + return JSONObject.toJSONString(response); + } + + @Override + public String videoGeneration(String dataJson) { + Map response = new HashMap<>(16); + int code = 200; + String message = "success"; + Map task = null; + try { + task = JSONObject.parseObject(dataJson); + } catch (Exception e) { + log.error("参数结构不合法，", e); + code = 100010; + message = "参数不合法"; + } + // 写入队列 + try { + if(task.containsKey(Constants.TRACE) && (boolean)task.get(Constants.TRACE)){ + ConfigCache.videoCommitTaskQueue.putFirst(task); + }else{ + ConfigCache.videoCommitTaskQueue.put(task); + } + } catch (InterruptedException e) { + log.error("任务写入等待队列异常，", e); + code = 100011; + message = "任务写入等待队列失败"; + } + response.put(Constants.CODE, code); + response.put(Constants.MESSAGE, message); + return JSONObject.toJSONString(response); + } + + +} diff --git a/src/main/java/com/bfd/qanda/utils/DataUtil.java b/src/main/java/com/bfd/qanda/utils/DataUtil.java new file mode 100644 index 0000000..6ca3666 --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/DataUtil.java @@ -0,0 +1,48 @@ +package com.bfd.qanda.utils; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSONPath; + +import java.util.Map; + +/** + * @author:jinming + * @className:DataUtil + * @version:1.0 + * @description: 获取dataValue的值 + * @Date:2023/11/1 9:54 + */ +public class DataUtil { + /** + * + * @param key 传入的key + * @param dataMap 数据map + * @return 根据传入的参数进行判断解析，返回正确的dataValue + */ + public static Object getValue(String key, Map dataMap) { + Object dataValue; + String isJson = "#json#"; + if (key.contains(isJson)) { + //进行第一次拆分，获取#json#前面的部分 + String[] keySplit = key.split(isJson); + String firstDataKey = keySplit[0]; + String[] firstDataKeySplit = firstDataKey.split(":"); + //取出前半部分对应的JSON数据并转换为JSONObject + String dataJson = (String) dataMap.get(firstDataKeySplit[0]); + JSONObject dataJsonObject = JSON.parseObject(dataJson); + //根据key的后半部分取出对应JSONObject中的值 + String firstDataKeyJson = (String) JSONPath.eval(dataJsonObject, firstDataKeySplit[1]); + String secDataKey = keySplit[1]; + JSONObject firstDataJsonObject = JSON.parseObject(firstDataKeyJson); + dataValue = JSONPath.eval(firstDataJsonObject, secDataKey); + return dataValue; + } + String[] keySplit = key.split(":"); + String jsonPath = keySplit[1]; + String dataJson = (String) dataMap.get(keySplit[0]); + JSONObject dataJsonObject = JSON.parseObject(dataJson); + dataValue = JSONPath.eval(dataJsonObject, jsonPath); + return dataValue; + } +} \ No newline at end of file diff --git a/src/main/java/com/bfd/qanda/utils/DateUtil.java b/src/main/java/com/bfd/qanda/utils/DateUtil.java new file mode 100644 index 0000000..0b42b01 --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/DateUtil.java @@ -0,0 +1,177 @@ +package com.bfd.qanda.utils; + + +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Date; + +import lombok.extern.slf4j.Slf4j; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; + +/** + * 日期工具类 + * + * @author jian.mao + * @date 2022年11月15日 + * @description + */ +@Slf4j +public class DateUtil { + + /** + * @return + */ + public static String getTimeStrForNow() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH"); + return sdf.format(new Date()); + } + + + public static String getTimeStrForDay(long time) { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); + + return sdf.format(new Date(time * 1000)); + } + + public static String getTimeStrForDay() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); + + return sdf.format(new Date()); + } + + + public static String getDateTime() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String time = sdf.format(new Date()); + return time; + } + + public static String getDateTime(Long timestap) { + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String time = sdf.format(new Date(timestap)); + return time; + } + + public static String getDate(Long timestap) { + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + String time = sdf.format(new Date(timestap)); + return time; + } + + public static String getDateTimeForMonth() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM"); + String time = sdf.format(new Date()); + return time; + } + + /** + * 休眠 + * + * @param millis 毫秒 + */ + public static void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + /** + * 1. @Description:时间戳转时间 + * 2. @Author: ying.zhao + * 3. @Date: 2023/3/28 + */ + + public static String timestampToDate(String time) { + int thirteen = 13; + int ten = 10; + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); +// if (time.length() == thirteen) { + if (time.length() > ten) { + return sdf.format(new Date(Long.parseLong(time))); + } else { + return sdf.format(new Date(Integer.parseInt(time) * 1000L)); + } + } + + public static String parseCreated(String jsonTime){ + String formattedDateTime = getDateTime(); + try { + // 使用fastjson解析JSON数据 + JSONObject jsonObject = JSON.parseObject(jsonTime); + // 获取日期和时间的值 + JSONObject dateObject = jsonObject.getJSONObject("date"); + int day = dateObject.getIntValue("day"); + int month = dateObject.getIntValue("month"); + int year = dateObject.getIntValue("year"); + + JSONObject timeObject = jsonObject.getJSONObject("time"); + int hour = timeObject.getIntValue("hour"); + int minute = timeObject.getIntValue("minute"); + int second = timeObject.getIntValue("second"); + + // 创建LocalDateTime对象 + LocalDateTime dateTime = LocalDateTime.of(year, month, day, hour, minute, second); + + // 定义日期时间格式化器 + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + // 格式化日期时间 + formattedDateTime = dateTime.format(formatter); + } catch (Exception e) { + log.info("日期转换失败：{}",e); + } + return formattedDateTime; + } + + /** + * 字符串转换日期 + * @param format + * @param date + * @return + */ + public static Date strToDate(String format,String date){ + SimpleDateFormat sdf = new SimpleDateFormat(format); + if (date == null || date.equals("")){ + return new Date(); + }else{ + Date ru = null; + try { + ru = sdf.parse(date); + } catch (ParseException e) { + e.printStackTrace(); + } + return ru; + } + } + /** + * 日期格式话 + * @param format 日期格式 + * @param dater 要转换的日期,默认当前时间 + * @return + */ + public static String FormatDate(String format,Date date){ + String fromatDate = null; + SimpleDateFormat sdf = new SimpleDateFormat(format); + if (date == null){ + fromatDate = sdf.format(new Date()); + }else{ + fromatDate = sdf.format(date); + } + return fromatDate; + } + public static void main(String[] args) { + String time = timestampToDate("955814400000"); + System.out.println(time); + } +} diff --git a/src/main/java/com/bfd/qanda/utils/DownLoadUtil.java b/src/main/java/com/bfd/qanda/utils/DownLoadUtil.java new file mode 100644 index 0000000..d0a9bec --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/DownLoadUtil.java @@ -0,0 +1,1070 @@ +package com.bfd.qanda.utils; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.StatusLine; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.AuthCache; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.HttpClient; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.LayeredConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.BasicAuthCache; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.LaxRedirectStrategy; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.bfd.qanda.entity.Constants; + + + + + + + +/** + * 下载工具类 + * @author jian.mao + * @date 2023年9月19日 + * @description + */ +public class DownLoadUtil { + + private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"; + private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class); + /** 代理服务器(产品官网 www.16yun.cn) **/ + final static String PROXYHOST = "u270.40.tp.16yun.cn"; + final static Integer PROXYPORT = 6448; + /** 代理验证信息 **/ + final static String PROXYUSER = "16HFBVJC"; + final static String PROXYPASS = "897944"; + + private static PoolingHttpClientConnectionManager cm = null; + private static HttpRequestRetryHandler httpRequestRetryHandler = null; + private static HttpHost proxy = null; + + private static CredentialsProvider credsProvider = null; + private static RequestConfig reqConfig = null; + + static { + ConnectionSocketFactory plainsf = PlainConnectionSocketFactory + .getSocketFactory(); + LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory + .getSocketFactory(); + + Registry registry = RegistryBuilder.create().register("http", plainsf) + .register("https", sslsf).build(); + + cm = new PoolingHttpClientConnectionManager(registry); + cm.setMaxTotal(20); + cm.setDefaultMaxPerRoute(5); + + proxy = new HttpHost(PROXYHOST, PROXYPORT, "https"); + + credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(PROXYUSER, PROXYPASS)); + + reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000) + .setConnectTimeout(5000).setSocketTimeout(5000) + .setExpectContinueEnabled(false) + .setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build(); + } + + /** + * 模拟客户端get请求 + * + * @param url + * 模拟请求得url + * @param headers + * 头部信息，没有可以不传 + * @return + */ + @SafeVarargs + public static String proxyDoGet(String url, Map... headers) { + // 设置超时时间 + int timeout = 30; + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) + .setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); + AuthCache authCache = new BasicAuthCache(); + authCache.put(proxy, new BasicScheme()); + HttpClientContext localContext = HttpClientContext.create(); + localContext.setAuthCache(authCache); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + CloseableHttpClient httpClient = httpBuilder + .setDefaultSocketConfig(socketConfig) + .setDefaultRequestConfig(config) + .setDefaultCredentialsProvider(credsProvider).build(); + HttpGet httpGet = new HttpGet(url); + httpGet.setConfig(reqConfig); + if (headers != null && headers.length > 0) { + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key, tempHeaders.get(key).toString()); + } + } else { + httpGet.setHeader("Accept", + "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + } + CloseableHttpResponse response = null; + String html = ""; + int notFundCode = 404; + int successCode = 200; + try { + response = httpClient.execute(httpGet, localContext); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + if (statusLine.getStatusCode() == successCode) { + if (responseEntity != null) { + html = EntityUtils.toString(responseEntity, "utf-8"); + System.out.println("响应内容长度为:" + + responseEntity.getContentLength()); + // 下载结果为空不正常 + if (html.equals(Constants.EMPTY)) { + html = "Download failed error is:reslut is null"; + } + } + } else if (statusLine.getStatusCode() == notFundCode) { + html = "

页面404，正常结束请求即可

"; + } else { + throw new Exception("请求错误，code码为：" + statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:reslut is null"; + }finally{ + try { + response.close(); + httpClient.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + return html; + + } + + + public static String httpsslProxyGet(String url, Map... headers) throws Exception { + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connManager.setMaxTotal(50); + connManager.setDefaultMaxPerRoute(10); + HttpClients.custom().setConnectionManager(connManager); + // 设置超时时间 + int timeout = 30; + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) + .setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); + AuthCache authCache = new BasicAuthCache(); + authCache.put(proxy, new BasicScheme()); + HttpClientContext localContext = HttpClientContext.create(); + localContext.setAuthCache(authCache); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + CloseableHttpClient httpClient = httpBuilder + .setConnectionManager(connManager) + .setDefaultSocketConfig(socketConfig) + .setDefaultRequestConfig(config) + .setDefaultCredentialsProvider(credsProvider).build(); + HttpGet httpGet = new HttpGet(url); + httpGet.setConfig(reqConfig); + if (headers != null && headers.length > 0) { + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key, tempHeaders.get(key).toString()); + } + } else { + httpGet.setHeader("Accept", + "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + } + CloseableHttpResponse response = null; + String html = ""; + int notFundCode = 404; + int successCode = 200; + try { + response = httpClient.execute(httpGet, localContext); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + if (statusLine.getStatusCode() == successCode) { + if (responseEntity != null) { + html = EntityUtils.toString(responseEntity, "utf-8"); + System.out.println("响应内容长度为:" + + responseEntity.getContentLength()); + // 下载结果为空不正常 + if (html.equals(Constants.EMPTY)) { + html = "Download failed error is:reslut is null"; + } + } + } else if (statusLine.getStatusCode() == notFundCode) { + html = "

页面404，正常结束请求即可

"; + } else { + throw new Exception("请求错误，code码为：" + statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:reslut is null"; + }finally{ + try { + response.close(); + httpClient.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + return html; + + } + + + /** + * json参数方式POST提交 + * @param url + * @param params + * @return + */ + public static String doPost(String url, String params, Map... headers){ + String strResult = ""; + //设置超时时间 + int timeout = 60; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); +// AuthCache authCache = new BasicAuthCache(); +// authCache.put(proxy, new BasicScheme()); +// HttpClientContext localContext = HttpClientContext.create(); +// localContext.setAuthCache(authCache); + // 1. 获取默认的client实例 + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); + HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); +// HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) +// .setDefaultCredentialsProvider(credsProvider).build(); + // 2. 创建httppost实例 + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + if (headers != null && headers.length > 0) { + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key, tempHeaders.get(key).toString()); + } + } else { + httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); + } + HttpResponse resp = null; + try { + httpPost.setEntity(new StringEntity(params,"utf-8")); + resp = client.execute(httpPost); +// resp = client.execute(httpPost,localContext); + StatusLine statusLine = resp.getStatusLine(); + System.out.println("响应状态为:" + resp.getStatusLine()); + int notFundCode = 404; + int successCode = 200; + if(statusLine.getStatusCode() == successCode){ + // 7. 获取响应entity + HttpEntity respEntity = resp.getEntity(); + strResult = EntityUtils.toString(respEntity, "UTF-8"); + if(strResult.equals(Constants.EMPTY)){ + strResult = "Download failed error is:reslut is null"; + } + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + return strResult; + } + public static String httpPost(String url,String params) { + String html=""; + html = doPost(url,params); + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(5000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + html = doPost(url,params); + } + return html; + } + /** + * 绕过验证 + * + * @return + * @throws NoSuchAlgorithmException + * @throws KeyManagementException + */ + public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException { + SSLContext sc = SSLContext.getInstance("SSLv3"); + + // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法 + X509TrustManager trustManager = new X509TrustManager() { + @Override + public void checkClientTrusted( + java.security.cert.X509Certificate[] paramArrayOfX509Certificate, + String paramString) throws CertificateException { + } + + @Override + public void checkServerTrusted( + java.security.cert.X509Certificate[] paramArrayOfX509Certificate, + String paramString) throws CertificateException { + } + + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + }; + + sc.init(null, new TrustManager[] { trustManager }, null); + return sc; + } + /** + * 模拟请求 + * + * @param url 资源地址 + * @param map 参数列表 + * @param encoding 编码 + * @return + * @throws NoSuchAlgorithmException + * @throws KeyManagementException + * @throws IOException + * @throws ClientProtocolException + */ + public static String httpsslGet(String url,Map ... headers) { + String html=""; + CloseableHttpClient client = null; + HttpEntity responseEntity = null; + CloseableHttpResponse response = null; + try { + log.debug("DownLoadUtil------------->设置下载相关信息， start...."); + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connManager.setMaxTotal(50); + connManager.setDefaultMaxPerRoute(10); + HttpClients.custom().setConnectionManager(connManager); + //设置超时时间 + int timeout = 30; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + // 设置重定向策略 + LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy(); + //创建自定义的httpclient对象 + client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build(); +// CloseableHttpClient client = HttpClients.createDefault(); + + HttpGet httpGet = new HttpGet(url); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); + } + log.debug("DownLoadUtil------------->设置下载相关信息， end...."); + try { + int notFundCode = 404; + int successCode = 200; + log.debug("DownLoadUtil------------->下载执行，start...."); + httpGet.setConfig(config); + response = client.execute(httpGet); + log.debug("DownLoadUtil------------->下载执行，end...."); + // 从响应模型中获取响应实体 + StatusLine statusLine = response.getStatusLine(); + log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is start ...."); + System.out.println("响应状态为:" + response.getStatusLine()); + responseEntity = response.getEntity(); + log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is end ...."); + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + System.out.println("响应内容长度为:" + responseEntity.getContentLength()); + } + }else if(statusLine.getStatusCode() == notFundCode){ + html = "

页面404，正常结束请求即可

"; + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + }finally{ + try { + responseEntity.getContent().close(); + response.close(); + client.close(); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + return html; + } + + public static String httpSSLGet(String url,Map ... headers) { + String html=""; + html = httpsslGet(url,headers); + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(30000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + html = httpsslGet(url,headers); + } + return html; + } + public static String doPostFrom(String url,Map param,Map ... headers){ + //设置超时时间 + int timeout = 15; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); +// AuthCache authCache = new BasicAuthCache(); +// authCache.put(proxy, new BasicScheme()); +// HttpClientContext localContext = HttpClientContext.create(); +// localContext.setAuthCache(authCache); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); +// HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) +// .setDefaultCredentialsProvider(credsProvider).build(); + HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"); + httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); + httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"); +// httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html"); + } + // 创建请求参数 + List list = new LinkedList<>(); + for (String key : param.keySet()) { + BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); + list.add(param1); + } + // 使用URL实体转换工具 + String html=""; + try { + UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); + httpPost.setEntity(entityParam); + HttpResponse response = httpClient.execute(httpPost); +// HttpResponse response = httpClient.execute(httpPost,localContext); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + int notFundCode = 404; + int successCode = 200; + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + } + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + + return html; + + } + public static String httpPostForm(String url,Map params,Map ... headers) { + String html=""; + html = doPostFrom(url,params); + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(5000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + html = doPostFrom(url,params,headers); + } + return html; + } + + public static String dosslPost(String url,String params,Map ... headers) { + String html=""; + CloseableHttpClient client = null; + HttpEntity responseEntity = null; + CloseableHttpResponse response = null; + try { + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + HttpClients.custom().setConnectionManager(connManager); + //设置超时时间 + int timeout = 5; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + //创建自定义的httpclient对象 + client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); +// CloseableHttpClient client = HttpClients.createDefault(); + // 2. 创建httppost实例 + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); + } + + try { + httpPost.setEntity(new StringEntity(params,"utf-8")); + response = client.execute(httpPost); + int notFundCode = 404; + int successCode = 200; + // 从响应模型中获取响应实体 + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + responseEntity = response.getEntity(); + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + System.out.println("响应内容长度为:" + responseEntity.getContentLength()); + } + }else if(statusLine.getStatusCode() == notFundCode){ + html = "

页面404，正常结束请求即可

"; + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + }finally{ + try { + responseEntity.getContent().close(); + response.close(); + client.close(); + } catch (UnsupportedOperationException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + + return html; + } + public static String dosslPostForm(String url,Map param,Map ... headers) { + String html=""; + try { + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + HttpClients.custom().setConnectionManager(connManager); + //设置超时时间 + int timeout = 5; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + //创建自定义的httpclient对象 + CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); +// CloseableHttpClient client = HttpClients.createDefault(); + // 2. 创建httppost实例 + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); + httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); + } + + // 创建请求参数 + List list = new LinkedList<>(); + for (String key : param.keySet()) { + BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); + list.add(param1); + } + // 使用URL实体转换工具 + try { + UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); + httpPost.setEntity(entityParam); + HttpResponse response = client.execute(httpPost); +// HttpResponse response = httpClient.execute(httpPost,localContext); + // 从响应模型中获取响应实体 + int notFundCode = 404; + int successCode = 200; + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + } + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + + + return html; + } + public static String httpSSLPostForm(String url,Map params,Map ...headers) { + String html=""; + try { + html = dosslPostForm(url,params,headers); + } catch (Exception e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(30000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + try { + html = dosslPostForm(url,params,headers); + } catch (Exception e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + } + return html; + } + public static String httpSSLPost(String url,String params,Map ...headers) { + String html=""; + try { + html = dosslPost(url,params,headers); + } catch (Throwable e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(30000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + try { + html = dosslPost(url,params,headers); + } catch (Throwable e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + } + return html; + } + + /** + * 模拟客户端get请求 + * @param url 模拟请求得url + * @param headers 头部信息，没有可以不传 + * @return + */ + public static String doGet(String url,Map ... headers){ + //设置超时时间 + int timeout = 15; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); + HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); + HttpGet httpGet = new HttpGet(url); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + } + String html=""; + try { + int notFundCode = 404; + int successCode = 200; + HttpResponse response = httpClient.execute(httpGet); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + if(html.equals("")){ + html = "Download failed error is:reslut is null"; + } + } + }else if(statusLine.getStatusCode() == notFundCode){ + html = "

页面404，正常结束请求即可

"; + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + return html; + + } + + /** + * 文件下载 + * @param fileURL 文件连接 + * @param saveFilePath 文件存储地址 + * @throws IOException + */ + public static void downloadFile(String fileURL, String saveFilePath,Map ... headers) { + //设置超时时间 + int timeout = 60; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); + HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); + HttpGet httpGet = new HttpGet(fileURL); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + } + try { + int successCode = 200; + HttpResponse response = httpClient.execute(httpGet); + // 从响应模型中获取响应实体 + StatusLine statusLine = response.getStatusLine(); + int statusCode = statusLine.getStatusCode(); + if(statusCode == successCode){ + // 获取响应体 + HttpEntity entity = response.getEntity(); + if (entity != null) { + // 创建输入流来读取文件内容 + try (InputStream inputStream = entity.getContent(); + FileOutputStream fileOutputStream = new FileOutputStream(new File(saveFilePath))) { + + byte[] buffer = new byte[4096]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + fileOutputStream.write(buffer, 0, bytesRead); + } + } + log.info("文件下载成功，保存路径为: {}" , saveFilePath); + } + } else { + log.error("文件下载失败，HTTP 响应状态码为: {}" , statusCode); + } + EntityUtils.consume(response.getEntity()); + } catch (Exception e) { + log.error("下载文件异常：{}",e); + } + + } + + /** + * 删除文件 + * @param filePath + */ + public static void delFile(String filePath) { + try { + // 创建 File 对象 + File file = new File(filePath); + // 检查文件是否存在 + if (file.exists()) { + // 尝试删除文件 + if (file.delete()) { + log.info("文件删除成功: " + filePath); + } else { + log.error("无法删除文件: " + filePath); + } + } else { + log.warn("文件不存在: " + filePath); + } + } catch (Exception e) { + // TODO: handle exception + log.error("删除文件异常：{}",filePath); + } + } + + /** + * @param uploadUrl + * @param filePath + */ + public static String uploadFile(String uploadUrl, String filePath) { + // 创建HttpClient实例 + CloseableHttpClient httpClient = HttpClients.createDefault(); + String gofastUrl = ""; + try { + // 创建HttpPost请求 + HttpPost httpPost = new HttpPost(uploadUrl); + // 要上传的文件 + File file = new File(filePath); + if (!file.exists()) { + log.error("文件不存在：{}" , filePath); + return gofastUrl; + } + // 构建文件上传请求体 + HttpEntity entity = MultipartEntityBuilder.create() + .addBinaryBody("file", file, ContentType.DEFAULT_BINARY, file.getName()) + .build(); + + // 设置请求体 + httpPost.setEntity(entity); + + // 执行请求并获取响应 + CloseableHttpResponse response = httpClient.execute(httpPost); + try { + int statusCode = response.getStatusLine().getStatusCode(); + int successCode = 200; + if (statusCode == successCode) { + // 解析响应内容 + gofastUrl = EntityUtils.toString(response.getEntity()); + log.info("文件上传成功，响应：{}" , gofastUrl); + } else { + log.error("文件上传失败，HTTP 响应码：{}" , statusCode); + } + }catch (Exception e) { + // TODO: handle exception + } finally { + try { + response.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + }catch(Exception e){ + e.printStackTrace(); + } finally { + try { + httpClient.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return gofastUrl; + } + public static void main(String[] args) throws Exception { +// String url = "https://public-cdn-video-data-algeng.oss-cn-wulanchabu.aliyuncs.com/inference_output%2Fvideo%2F2024-09-14%2Fa13bd277-691e-4a3e-9096-ec5e9a449aa3%2Foutput.mp4?Expires=1726313297&OSSAccessKeyId=LTAI5tGLnRTkBjLuYPjNcKQ8&Signature=TmZDbYyisMoNi0smdEUCkzooTZI%3D"; + String path = "D:\\123.mp4"; +// downloadFile(url,path); + String uploadUrl = "http://172.18.1.180:9980/group17/upload"; + uploadFile(uploadUrl, path); + } +} diff --git a/src/main/java/com/bfd/qanda/utils/EncryptionUtil.java b/src/main/java/com/bfd/qanda/utils/EncryptionUtil.java new file mode 100644 index 0000000..1efdef6 --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/EncryptionUtil.java @@ -0,0 +1,27 @@ +package com.bfd.qanda.utils; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * @author jian.mao + * @date 2023年3月10日 + * @description + */ +public class EncryptionUtil { + public static String md5(String text) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(text.getBytes()); + byte[] bytes = md.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b & 0xff)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + return null; + } + } +} diff --git a/src/main/java/com/bfd/qanda/utils/FileUtil.java b/src/main/java/com/bfd/qanda/utils/FileUtil.java new file mode 100644 index 0000000..7e40c19 --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/FileUtil.java @@ -0,0 +1,36 @@ +package com.bfd.qanda.utils; + +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 文件工具类 + * @author jian.mao + * @date 2023年7月14日 + * @description + */ +public class FileUtil { + + /** + * 数据写入文件 + * @param Path 文件路径 + * @param result 数据 + * @throws IOException + */ + public static void writeFile(String path,String result){ + try { + FileWriter fw = new FileWriter(path,true); + fw.write(result+"\n"); + fw.flush(); + fw.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java b/src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java new file mode 100644 index 0000000..3bd86df --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java @@ -0,0 +1,53 @@ +package com.bfd.qanda.utils; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSONException; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author:jinming + * @className:GPTResultParseUtil + * @version:1.0 + * @description: + * @Date:2024/6/28 10:11 + */ +public class GPTResultParseUtil { + public static Map parseGPTResult(Map output, String gptContent) { + Map jsonResult = new HashMap<>(); + try { + // 替换```json, ``` 和 \n + String jsonContent = gptContent.replace("```json", "").replace("```", "").replace("\n", ""); + JSONObject jsonGPT = JSON.parseObject(jsonContent); + + for (String key : output.keySet()) { + if (jsonGPT.containsKey(key)) { + jsonResult.put(key, jsonGPT.get(key)); + } + } + return jsonResult; + } catch (JSONException e) { + try { + // 直接解析失败，使用正则表达式匹配外层的 {} + Pattern pattern = Pattern.compile("\\{.*\\}", Pattern.DOTALL); + Matcher matcher = pattern.matcher(gptContent.replace("\n", "")); + if (matcher.find()) { + JSONObject jsonGPT = JSON.parseObject(matcher.group()); + for (String key : output.keySet()) { + if (jsonGPT.containsKey(key)) { + jsonResult.put(key, jsonGPT.get(key)); + } + } + return jsonResult; + } else { + return null; + } + } catch (Exception ex) { + ex.printStackTrace(); + return null; + } + } + } +} \ No newline at end of file diff --git a/src/main/java/com/bfd/qanda/utils/JsonUtil.java b/src/main/java/com/bfd/qanda/utils/JsonUtil.java new file mode 100644 index 0000000..46fb9fb --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/JsonUtil.java @@ -0,0 +1,32 @@ +package com.bfd.qanda.utils; + +import com.alibaba.fastjson.JSONObject; +import com.bfd.qanda.entity.Constants; + +/** + * json工具 + * @author jian.mao + * @date 2023年7月10日 + * @description + */ +public class JsonUtil { + + /** + * 校验字符串是list/map/str + * @param jsonString + * @return + */ + public static String checkJsonType(String jsonString) { + try { + JSONObject.parseObject(jsonString); + return Constants.MAP_TYPE; + } catch (Exception e) { + try { + JSONObject.parseArray(jsonString); + return Constants.LIST_TYPE; + } catch (Exception ex) { + return Constants.STRING_TYPE; + } + } + } +} diff --git a/src/main/java/com/bfd/qanda/utils/OtherUtils.java b/src/main/java/com/bfd/qanda/utils/OtherUtils.java new file mode 100644 index 0000000..0d8d9b2 --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/OtherUtils.java @@ -0,0 +1,33 @@ +package com.bfd.qanda.utils; + +import java.security.MessageDigest; + +/** + * 其他工具类 + * @author jian.mao + * @date 2023年9月19日 + * @description + */ +public class OtherUtils { + + + + public static String getMd5(String string) { + try { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + byte[] bs = md5.digest(string.getBytes("UTF-8")); + StringBuilder sb = new StringBuilder(40); + for (byte x : bs) { + if ((x & 0xff) >> 4 == 0) { + sb.append("0").append(Integer.toHexString(x & 0xff)); + } else { + sb.append(Integer.toHexString(x & 0xff)); + } + } + return sb.toString(); + } catch (Exception e) { + + return "nceaform" + System.currentTimeMillis(); + } + } +} diff --git a/src/main/java/com/bfd/qanda/utils/PauseTool.java b/src/main/java/com/bfd/qanda/utils/PauseTool.java new file mode 100644 index 0000000..b9f30bd --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/PauseTool.java @@ -0,0 +1,92 @@ +package com.bfd.qanda.utils; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.cache.NodeCache; +import org.apache.curator.framework.recipes.cache.NodeCacheListener; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.util.HashMap; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author:jinming + * @className:ZookeeperNodeMonitor + * @version:1.0 + * @description: Zookeeper节点监听和Redis初始化工具类 + * @Date:2024/7/2 14:20 + */ +@Component +@Slf4j +public class PauseTool { + + // 本地缓存 + public static final HashMap CACHE = new HashMap<>(); + + /** + * 初始化Redis中的version_*键并加载到本地缓存 + */ + public void initializeRedisCache(StringRedisTemplate stringRedisTemplate) { + try { + Set keys = stringRedisTemplate.keys("version_*"); + if (keys != null) { + for (String key : keys) { + String value = stringRedisTemplate.opsForValue().get(key); + if (value != null) { + String sincesId = key.split("_")[1]; + CACHE.put(sincesId.concat("_").concat(value), value); + } + } + } + log.info("当前缓存version信息:{}", JSON.toJSON(CACHE)); + } catch (Exception e) { + log.error("Error initializing Redis cache", e); + } + } + + public void setupZookeeperListener(String connectionString, String nodePath) { + CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(connectionString, new ExponentialBackoffRetry(1000, 3)); + curatorFramework.start(); + try { + // 创建节点监听器 + NodeCache nodeCache = new NodeCache(curatorFramework, nodePath); + nodeCache.start(); + log.info("数据监听已启动"); + // 监听节点变化 + nodeCache.getListenable().addListener(new NodeCacheListener() { + @Override + public void nodeChanged() throws Exception { + byte[] data = nodeCache.getCurrentData().getData(); + try { + String nodeData = new String(data); + log.info("Node data changed: " + nodeData); + // 解析JSON数据 + JSONObject jsonObject = JSON.parseObject(nodeData); + int scenesId = jsonObject.getIntValue("scenes_id"); + int version = jsonObject.getIntValue("version"); + String newKey = scenesId + "_" + version; + // 移除CACHE中所有以scenesId开头的key + CACHE.keySet().removeIf(key -> key.startsWith(scenesId + "_")); + // 将新的key放入CACHE + CACHE.put(newKey, String.valueOf(version)); + log.info("当前缓存version信息:{}", JSON.toJSON(CACHE)); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + } catch (Exception e) { + log.error("Error setting up Zookeeper listener", e); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/bfd/qanda/utils/QueueUtil.java b/src/main/java/com/bfd/qanda/utils/QueueUtil.java new file mode 100644 index 0000000..5796c2a --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/QueueUtil.java @@ -0,0 +1,18 @@ +package com.bfd.qanda.utils; + +import java.util.Map; +import java.util.concurrent.LinkedBlockingDeque; + +/** + * @author:jinming + * @className:QueueUtil + * @version:1.0 + * @description: + * @Date:2023/7/13 15:00 + */ +public class QueueUtil { + + public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>(); + + public static LinkedBlockingDeque sendQueue = new LinkedBlockingDeque(); +} \ No newline at end of file diff --git a/src/main/java/com/bfd/qanda/utils/SpringBootKafka.java b/src/main/java/com/bfd/qanda/utils/SpringBootKafka.java new file mode 100644 index 0000000..be8ad73 --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/SpringBootKafka.java @@ -0,0 +1,46 @@ +package com.bfd.qanda.utils; + +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.SendResult; +import org.springframework.stereotype.Component; +import org.springframework.util.concurrent.ListenableFuture; +import org.springframework.util.concurrent.ListenableFutureCallback; + +/** + * @PROJECT_NAME: companybusinesscrawl + * @DESCRIPTION:SpringBootKafka 工具类 + * @AUTHOR: ying.zhao + * @DATE: 2023/4/6 11:09 + */ +@Slf4j +@Component +public class SpringBootKafka { + @Autowired + private KafkaTemplate kafkaTemplate; + /** + * 自定义topicKafkaTemplate + */ + /** + * public static final String TOPIC = "companyBussTest"; + **/ + public void send(String topic, String message) { + //发送消息 + ListenableFuture> future = kafkaTemplate.send(topic, message); + future.addCallback(new ListenableFutureCallback>() { + @Override + public void onFailure(Throwable throwable) { + //发送失败的处理 + log.info(topic + " - 生产者 发送消息失败：" + throwable.getMessage()); + } + + @Override + public void onSuccess(SendResult stringObjectSendResult) { + //成功的处理 + log.info(topic + " - 生产者 发送消息成功" ); + } + }); + } +} diff --git a/src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java b/src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java new file mode 100644 index 0000000..8771354 --- /dev/null +++ b/src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java @@ -0,0 +1,23 @@ +package com.bfd.qanda.utils; + +import java.io.PrintWriter; +import java.io.StringWriter; + +/** + * @author jian.mao + * @date 2023年3月22日 + * @description + */ +public class ThrowMessageUtil { + + /** + * 获取异常信息 + * @param t + * @return + */ + public static String getErrmessage(Throwable t){ + StringWriter stringWriter=new StringWriter(); + t.printStackTrace(new PrintWriter(stringWriter,true)); + return stringWriter.getBuffer().toString(); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..26257b1 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,102 @@ +logging: + level: + root: info + path: ./logs +server: + port: 8020 + servlet: + context-path: /minimaxmodel + tomcat: + uri-encoding: utf-8 + max-connections: 20000 + max-http-form-post-size: 1 + max-threads: 1000 +spring: + application: + name: bigmodel + kafka: + bootstrap-servers: 172.26.28.30:9092 + producer: + retries: 0 + #当有多个消息需要被发送到同一个分区时，生产者会把它们放在同一个批次里。该参数指定了一个批次可以使用的内存大小，按照字节数计算。 + batch-size: 16384 + # 设置生产者内存缓冲区的大小。 + buffer-memory: 33554432 + # 键的序列化方式 + key-serializer: org.apache.kafka.common.serialization.StringSerializer + # 值的序列化方式 + value-serializer: org.apache.kafka.common.serialization.StringSerializer + # acks=0 ： 生产者在成功写入消息之前不会等待任何来自服务器的响应。 + # acks=1 ： 只要集群的首领节点收到消息，生产者就会收到一个来自服务器成功响应。 + # acks=all ：只有当所有参与复制的节点全部收到消息时，生产者才会收到一个来自服务器的成功响应。 + acks: 1 + consumer: + # 自动提交的时间间隔 在spring boot 2.X 版本中这里采用的是值的类型为Duration 需要符合特定的格式，如1S,1M,2H,5D + auto-commit-interval: 1S + # 该属性指定了消费者在读取一个没有偏移量的分区或者偏移量无效的情况下该作何处理： + # latest（默认值）在偏移量无效的情况下，消费者将从最新的记录开始读取数据（在消费者启动之后生成的记录） + # earliest ：在偏移量无效的情况下，消费者将从起始位置读取分区的记录 + auto-offset-reset: earliest + # 是否自动提交偏移量，默认值是true,为了避免出现重复数据和数据丢失，可以把它设置为false,然后手动提交偏移量 + enable-auto-commit: true + # 键的反序列化方式 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + # 值的反序列化方式 + value-deserializer: org.apache.kafka.common.serialization.StringDeserializer + #消费组 + group-id: test4 + #消费者并发线程数 + concurrency: 4 + #超时时间 + max-poll-interval-ms: 60000 + #listener: + # 在侦听器容器中运行的线程数。 + #concurrency: 5 + #listner负责ack，每调用一次，就立即commit + #ack-mode: manual_immediate + #missing-topics-fatal: false + + boot: + admin: + client: + url: http://172.16.12.55:8001 + instance: + service-base-url: http://10.10.143.85:8010 + redis: + host: 172.24.12.126 + port: 6379 + timeout: 10000 + database: 5 + jedis: + pool: + max-active: 8 # 连接池最大连接数（使用负值表示没有限制） + max-wait: 800 # 连接池最大阻塞等待时间（使用负值表示没有限制） + max-idle: 8 # 连接池中的最大空闲连接 + min-idle: 2 # 连接池中的最小空闲连接 +management: + endpoints: + web: + exposure: + include: "*" + endpoint: + health: + show-details: always + health: + elasticsearch: + enabled: false + +customize-kafka: + bootstrap-servers: 172.18.1.119:9992 + producer: + topic: analyze +task: + task-queue-path: ../data/taskQueue.txt + thread-num: 1 +zookeeper: + connection-string: 172.18.1.146:2181,172.18.1.147:2181,172.18.1.148:2181 + publish-node: /analyze + +file: + path: ../files/ +gofast: + url: http://172.18.1.180:9980/group17/upload \ No newline at end of file diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..fd26abb --- /dev/null +++ b/src/main/resources/logback-spring.xml @@ -0,0 +1,36 @@ + + + + + + + + + true + + ${logging.level} + + + ${logging.path}/minimaxmodelInfo.log + + + ${logging.path}/minimaxmodelInfo.log.%d{yyyy-MM-dd} + 7 + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n + UTF-8 + + + + + + + + diff --git a/src/test/java/com/bfd/AppTest.java b/src/test/java/com/bfd/AppTest.java new file mode 100644 index 0000000..7f816c2 --- /dev/null +++ b/src/test/java/com/bfd/AppTest.java @@ -0,0 +1,20 @@ +package com.bfd; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Unit test for simple App. + */ +public class AppTest +{ + /** + * Rigorous Test :-) + */ + @Test + public void shouldAnswerWithTrue() + { + assertTrue( true ); + } +}