From 72f1b3775e9eec669d7fe18e29b4f3770c10655c Mon Sep 17 00:00:00 2001
From: 55007 <55007@maojian>
Date: Tue, 7 Jan 2025 16:38:43 +0800
Subject: [PATCH] =?UTF-8?q?=E6=99=BA=E6=99=AE=E5=A4=A7=E6=A8=A1=E5=9E=8B?=
=?UTF-8?q?=E5=BA=94=E7=94=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.classpath | 40 +
.gitignore | 2 +
.project | 23 +
.settings/org.eclipse.core.resources.prefs | 5 +
.settings/org.eclipse.jdt.core.prefs | 9 +
.settings/org.eclipse.m2e.core.prefs | 4 +
README.md | 1 +
pom.xml | 260 ++++++
src/main/java/com/bfd/qanda/Application.java | 24 +
src/main/java/com/bfd/qanda/cache/ConfigCache.java | 35 +
src/main/java/com/bfd/qanda/cache/LocalCache.java | 65 ++
.../com/bfd/qanda/controller/QandaController.java | 41 +
src/main/java/com/bfd/qanda/entity/Constants.java | 152 ++++
.../java/com/bfd/qanda/handler/MainHandler.java | 128 +++
.../java/com/bfd/qanda/process/QandaPorcess.java | 193 +++++
.../java/com/bfd/qanda/service/QandaService.java | 17 +
.../bfd/qanda/service/impl/QandaServiceImpl.java | 58 ++
.../java/com/bfd/qanda/token/TokenManagerV3.java | 93 +++
src/main/java/com/bfd/qanda/utils/DataUtil.java | 48 ++
src/main/java/com/bfd/qanda/utils/DateUtil.java | 177 ++++
.../java/com/bfd/qanda/utils/DownLoadUtil.java | 914 +++++++++++++++++++++
.../java/com/bfd/qanda/utils/EncryptionUtil.java | 27 +
src/main/java/com/bfd/qanda/utils/FileUtil.java | 36 +
.../com/bfd/qanda/utils/GPTResultParseUtil.java | 53 ++
src/main/java/com/bfd/qanda/utils/JsonUtil.java | 32 +
src/main/java/com/bfd/qanda/utils/OtherUtils.java | 33 +
src/main/java/com/bfd/qanda/utils/QueueUtil.java | 18 +
.../java/com/bfd/qanda/utils/SpringBootKafka.java | 46 ++
.../java/com/bfd/qanda/utils/ThrowMessageUtil.java | 23 +
src/main/resources/application.yml | 85 ++
src/main/resources/logback-spring.xml | 36 +
src/test/java/com/bfd/AppTest.java | 53 ++
32 files changed, 2731 insertions(+)
create mode 100644 .classpath
create mode 100644 .gitignore
create mode 100644 .project
create mode 100644 .settings/org.eclipse.core.resources.prefs
create mode 100644 .settings/org.eclipse.jdt.core.prefs
create mode 100644 .settings/org.eclipse.m2e.core.prefs
create mode 100644 README.md
create mode 100644 pom.xml
create mode 100644 src/main/java/com/bfd/qanda/Application.java
create mode 100644 src/main/java/com/bfd/qanda/cache/ConfigCache.java
create mode 100644 src/main/java/com/bfd/qanda/cache/LocalCache.java
create mode 100644 src/main/java/com/bfd/qanda/controller/QandaController.java
create mode 100644 src/main/java/com/bfd/qanda/entity/Constants.java
create mode 100644 src/main/java/com/bfd/qanda/handler/MainHandler.java
create mode 100644 src/main/java/com/bfd/qanda/process/QandaPorcess.java
create mode 100644 src/main/java/com/bfd/qanda/service/QandaService.java
create mode 100644 src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java
create mode 100644 src/main/java/com/bfd/qanda/token/TokenManagerV3.java
create mode 100644 src/main/java/com/bfd/qanda/utils/DataUtil.java
create mode 100644 src/main/java/com/bfd/qanda/utils/DateUtil.java
create mode 100644 src/main/java/com/bfd/qanda/utils/DownLoadUtil.java
create mode 100644 src/main/java/com/bfd/qanda/utils/EncryptionUtil.java
create mode 100644 src/main/java/com/bfd/qanda/utils/FileUtil.java
create mode 100644 src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java
create mode 100644 src/main/java/com/bfd/qanda/utils/JsonUtil.java
create mode 100644 src/main/java/com/bfd/qanda/utils/OtherUtils.java
create mode 100644 src/main/java/com/bfd/qanda/utils/QueueUtil.java
create mode 100644 src/main/java/com/bfd/qanda/utils/SpringBootKafka.java
create mode 100644 src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java
create mode 100644 src/main/resources/application.yml
create mode 100644 src/main/resources/logback-spring.xml
create mode 100644 src/test/java/com/bfd/AppTest.java
diff --git a/.classpath b/.classpath
new file mode 100644
index 0000000..1a0a8d2
--- /dev/null
+++ b/.classpath
@@ -0,0 +1,40 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..91eac80
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+/target/
+/logs/
\ No newline at end of file
diff --git a/.project b/.project
new file mode 100644
index 0000000..f022afc
--- /dev/null
+++ b/.project
@@ -0,0 +1,23 @@
+
+
+ bigmodel
+
+
+
+
+
+ org.eclipse.jdt.core.javabuilder
+
+
+
+
+ org.eclipse.m2e.core.maven2Builder
+
+
+
+
+
+ org.eclipse.jdt.core.javanature
+ org.eclipse.m2e.core.maven2Nature
+
+
diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs
new file mode 100644
index 0000000..839d647
--- /dev/null
+++ b/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,5 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//src/test/java=UTF-8
+encoding/=UTF-8
diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..71df522
--- /dev/null
+++ b/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,9 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.methodParameters=generate
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
+org.eclipse.jdt.core.compiler.compliance=1.8
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
+org.eclipse.jdt.core.compiler.release=disabled
+org.eclipse.jdt.core.compiler.source=1.8
diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs
new file mode 100644
index 0000000..f897a7f
--- /dev/null
+++ b/.settings/org.eclipse.m2e.core.prefs
@@ -0,0 +1,4 @@
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..72c909d
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+智谱大模型应用
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..f7fb655
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,260 @@
+
+
+
+ 4.0.0
+
+ org.springframework.boot
+ spring-boot-starter-parent
+ 2.2.4.RELEASE
+
+ com.bfd
+ bigmodel
+ 0.0.1-SNAPSHOT
+
+ bigmodel
+
+ http://www.example.com
+
+
+ UTF-8
+ 1.8
+ 1.8
+
+
+
+
+ junit
+ junit
+ 4.11
+ test
+
+
+ org.springframework.boot
+ spring-boot-starter-web
+
+
+
+ de.codecentric
+ spring-boot-admin-starter-client
+ 2.2.4
+
+
+ com.google.code.gson
+ gson
+ 2.8.8
+
+
+ org.springframework.boot
+ spring-boot-test
+
+
+
+ org.springframework
+ spring-test
+ 5.0.10.RELEASE
+ test
+
+
+ commons-io
+ commons-io
+ 1.4
+
+
+ com.alibaba
+ fastjson
+ 2.0.17
+
+
+
+ com.mchange
+ c3p0
+ 0.9.5.5
+
+
+ mysql
+ mysql-connector-java
+ 8.0.29
+
+
+
+ com.squareup.okhttp3
+ okhttp
+ 4.9.3
+
+
+ org.apache.httpcomponents
+ httpclient
+ 4.5.3
+
+
+ commons-lang
+ commons-lang
+ 2.6
+
+
+
+ org.jetbrains.kotlin
+ kotlin-reflect
+ 1.6.21
+ runtime
+
+
+
+ org.jsoup
+ jsoup
+ 1.8.1
+
+
+
+ org.projectlombok
+ lombok
+
+
+ org.springframework.kafka
+ spring-kafka
+
+
+
+
+ cn.hutool
+ hutool-all
+ 5.8.5
+
+
+ junit
+ junit
+
+
+
+ p6spy
+ p6spy
+ 3.9.0
+
+
+
+ commons-collections
+ commons-collections
+ 3.2.2
+
+
+ org.elasticsearch.client
+ transport
+ 6.0.0
+
+
+
+ org.elasticsearch
+ elasticsearch
+ 6.0.0
+
+
+ org.elasticsearch.plugin
+ transport-netty4-client
+ 6.0.0
+
+
+
+ com.auth0
+ java-jwt
+ 4.2.2
+
+
+ com.bfd.util
+ pauseTool
+ 1.0
+
+
+
+
+
+
+
+
+ maven-clean-plugin
+ 3.1.0
+
+
+
+ maven-resources-plugin
+ 3.0.2
+
+
+ maven-compiler-plugin
+ 3.8.0
+
+
+ maven-surefire-plugin
+ 2.22.1
+
+
+ maven-jar-plugin
+ 3.0.2
+
+
+ maven-install-plugin
+ 2.5.2
+
+
+ maven-deploy-plugin
+ 2.8.2
+
+
+
+ maven-site-plugin
+ 3.7.1
+
+
+ maven-project-info-reports-plugin
+ 3.0.0
+
+
+
+
+ org.springframework.boot
+ spring-boot-maven-plugin
+
+ com.bfd.qanda.Application
+ ZIP
+
+
+ ${project.groupId}
+ ${project.artifactId}
+
+
+
+
+
+
+ repackage
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+ 3.1.1
+
+
+ copy
+ package
+
+ copy-dependencies
+
+
+ jar
+ jar
+ runtime
+ ${project.build.directory}/libs
+
+
+
+
+
+
+
+
+
diff --git a/src/main/java/com/bfd/qanda/Application.java b/src/main/java/com/bfd/qanda/Application.java
new file mode 100644
index 0000000..9c1d177
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/Application.java
@@ -0,0 +1,24 @@
+package com.bfd.qanda;
+
+
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.kafka.annotation.EnableKafka;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+/**
+ * 主入口
+ *
+ * @author jian.mao
+ * @date 2023年7月4日
+ * @description
+ */
+@SpringBootApplication
+@EnableKafka
+public class Application {
+
+ public static void main(String[] args) {
+ SpringApplication.run(Application.class, args);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/qanda/cache/ConfigCache.java b/src/main/java/com/bfd/qanda/cache/ConfigCache.java
new file mode 100644
index 0000000..383daac
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/cache/ConfigCache.java
@@ -0,0 +1,35 @@
+package com.bfd.qanda.cache;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingDeque;
+
+/**
+ * @author jian.mao
+ * @date 2022年11月11日
+ * @description 静态变量类
+ */
+@Slf4j
+public class ConfigCache {
+
+ /**启动条件**/
+ public static boolean isStart = true;
+ /*****任务队列*****/
+ public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>();
+
+
+ /**
+ * 队列录入任务
+ * @param queue
+ * @param task
+ */
+ public static void putQueue(LinkedBlockingDeque> queue,Map task){
+ //next app 写入队列准备调出
+ try {
+ queue.put(task);
+ } catch (InterruptedException e) {
+ log.error("队列写入data失败---");
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/cache/LocalCache.java b/src/main/java/com/bfd/qanda/cache/LocalCache.java
new file mode 100644
index 0000000..c692107
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/cache/LocalCache.java
@@ -0,0 +1,65 @@
+package com.bfd.qanda.cache;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * token 缓存
+ * @author jian.mao
+ * @date 2023年11月8日
+ * @description
+ */
+@Slf4j
+public class LocalCache {
+
+ private static final ConcurrentMap CACHE = new ConcurrentHashMap<>(64);
+
+ private LocalCache() {
+ }
+
+ public static LocalCache getInstance() {
+ return Inner.LOCAL_CACHE;
+ }
+
+ public static String get(String key) {
+ Value v = LocalCache.CACHE.get(key);
+ if (v == null || new Date().after(v.end)) {
+ return "";
+ }
+
+ log.debug("get key:{},time left:{}s", key, (v.end.getTime() - new Date().getTime()) / 1000);
+ return v.value;
+ }
+
+ public static void set(String key, String value, int expire, TimeUnit timeUnit) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.add(Calendar.SECOND, (int) timeUnit.toSeconds(expire));
+ Value v = new Value(value, calendar.getTime());
+ log.debug("put key:{}, expire time:{} ", key, calendar.getTime());
+ LocalCache.CACHE.put(key, v);
+ }
+
+ private static class Value {
+
+ String value;
+ Date end;
+
+ public Value(String value, Date time) {
+ this.value = value;
+ this.end = time;
+ }
+ }
+
+ private static class Inner {
+
+ private static final LocalCache LOCAL_CACHE = new LocalCache();
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/controller/QandaController.java b/src/main/java/com/bfd/qanda/controller/QandaController.java
new file mode 100644
index 0000000..a0a44a8
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/controller/QandaController.java
@@ -0,0 +1,41 @@
+package com.bfd.qanda.controller;
+
+import javax.annotation.Resource;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.bfd.qanda.service.QandaService;
+
+
+/**
+ * @author jian.mao
+ * @date 2023年11月8日
+ * @description
+ */
+@Controller
+@RequestMapping("/qanda")
+@Slf4j
+public class QandaController {
+
+ @Resource
+ private QandaService qandaService;
+ @PostMapping("/putQuestion")
+ @ResponseBody
+ public String putQuestion(@RequestBody String dataJson){
+ String response = qandaService.putQuestion(dataJson);
+ return response;
+ }
+
+ @RequestMapping(value = "/hello", method = RequestMethod.GET)
+ @ResponseBody
+ public String hello(String param, String token) {
+ return "123";
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/entity/Constants.java b/src/main/java/com/bfd/qanda/entity/Constants.java
new file mode 100644
index 0000000..2e5b416
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/entity/Constants.java
@@ -0,0 +1,152 @@
+package com.bfd.qanda.entity;
+
+
+/**
+ * 常量实体类
+ * @author jian.mao
+ * @date 2022年11月15日
+ * @description
+ */
+public class Constants {
+
+ /*************************蓝图常量key名称*********************************/
+ public final static String SCHEDULING = "scheduling";
+ public final static String TYPE = "type";
+ public final static String INTERVAL = "interval";
+ public final static String CREATED = "created";
+ public final static String LAST_EDIT = "last_edit";
+ public final static String BLUEPRINT_ID = "blueprint_id";
+ public final static String BLUEPRINTID = "blueprintId";
+ public final static String BLUEPRINT_NAME = "name";
+ public final static String SCENARIO = "scenario";
+ public final static String AUTOCOMMITTRIGGERLAST = "autoCommitTriggerLast";
+ public final static String FRESHVARIABLES = "freshVariables";
+ public final static String AUTOCOMMIT = "autoCommit";
+ public final static String MAXERRORS = "maxErrors";
+ public final static String DATALOSS = "dataloss";
+ public final static String POSITION = "position";
+ public final static String SCENES_ID = "scenes_id";
+ public final static String SCENESID = "scenesId";
+ public final static String MULTI_BRANCH = "multi_branch";
+
+ public final static String SINGLE = "single";
+ /**已重试次数**/
+ public final static String ERROR_TIME = "error_time";
+ public final static String PREVIOUS_RESULT = "previous_result";
+
+ /****数据id*****/
+ public final static String BUSINESSKEY = "businessKey";
+
+
+ /*************************metadata常量key名称*********************************/
+ public final static String LABEL_COL = "label_col";
+ public final static String LABEL = "label";
+ public final static String USER = "user";
+ public final static String ADMIN = "admin";
+ public final static String ADDRESS = "address";
+ public final static String DATASOURCE = "datasource";
+ public final static String INDEX = "index";
+
+ /*************************app常量key名称*********************************/
+ public final static String APPS = "apps";
+ public final static String TRANSFER_ID = "transfer_id";
+ public final static String MODULE = "module";
+ public final static String VERSION = "version";
+ public final static String METADATA = "metadata";
+ public final static String APP_NAME = "name";
+ public final static String DESCRIBE = "describe";
+ public final static String NEXT_APP_ID = "next_app_id";
+ public final static String EDGE_ID = "edge_id";
+ public final static String START_ID = "start_id";
+ public final static String END_ID = "end_id";
+
+ public final static String WAIT_CONDITION = "wait_condition";
+ public final static String START_TAG = "start_tag";
+
+ /*************************module类型*********************************/
+ public final static String FILE = "file";
+ public final static String OCR = "OCR";
+ public final static String FILTER = "Filter";
+ public final static String CHATGPT = "ChatGPT";
+ public final static String MYSQL = "mysql";
+
+ /*************************other类型*********************************/
+ public final static String UNDERLINE = "_";
+ public final static String RESULT_TOPIC = null;
+ public static final String EMPTY = "";
+ public static final String HTTP = "http";
+ public static final String REQUEST_ERROR_MESSAGE = "Download failed error is";
+ public static final String REQUEST_RESULT = "result";
+ public static final String REQUEST_RESULT_RESULTS = "results";
+ public static final String MAP_TYPE = "Map";
+ public static final String LIST_TYPE = "List";
+ public static final String STRING_TYPE = "String";
+ public static final String DOCUMENT_TYPE = "doc";
+ public static final String FILTER_ZH = "过滤器";
+
+ public static final String JSON_SELE_SYMBOL = "$.";
+ public static final String LEFT_BRACKETS = "[";
+ public static final String RIGTH_BRACKETS = "]";
+ public static final String TASKTYPE = "taskType";
+ public static final Integer USER_TYPE = 1;
+ public static final Integer KEYWORD_TYPE = 0;
+ public static final Integer DETAIL_TYPE = 2;
+ public static final String CID = "cid";
+ public static final String SITETYPE = "siteType";
+ public static final Integer DEFULT_SUBJECTID = 304864;
+ public static final Integer DEFULT_CRAWLCYCLICITYTIME = 1440;
+ public static final String CRAWLENDTIME = "crawlEndTime";
+ public static final String CRAWLSTARTTIME = "crawlStartTime";
+ public static final String CRAWLPAGETYPES = "crawlPageTypes";
+ public static final String APPID = "113ic";
+ public static final String APP_ID = "appId";
+ public final static String ID = "id";
+ public static final Integer DEFULT_CRAWLPERIODHOUR = 24;
+ public static final String CREATEUSERID = "662015832180933762";
+ public static final String CRAWL_ADD_URL = "https://caiji.percent.cn/api/crawl/remote/task/save";
+ public static final String CRAWLKEYWORD = "crawlKeyword";
+ public static final String ATTACHTAG = "attachTag";
+ public static final String ATTACHTAG_VALUE = "analyze";
+ public static final String KEYWORD = "keyword";
+ public static final String SITEID = "siteId";
+ public static final String RESULTS = "results";
+ public static final String RESULT = "result";
+ public static final String CRAWLDATAFLAG = "crawlDataFlag";
+ public static final String CRAWLDATAFLAG_PREFIX = "\"crawlDataFlag\":\"keyword:";
+ public static final String TID = "tid";
+ public static final Long TIME_OUT = 1800000L;
+ public static final String ATTR = "attr";
+ public static final String HASVIDEO = "hasVideo";
+ public static final String CRAWL_END_MARK = "crawl_end_mark";
+ public static final String CRAWL_END_MESSAGE = "crawl_end_message";
+ public static final String CRAWL_END_MESSAGE_VALUE = "数据采集完成";
+ public static final String SUBJECTID = "subjectId";
+ public static final String TASKID = "taskId";
+ public static final int SUCCESS_CODE = 200;
+ public static final String WEB_URL_SUFFIX = "/api/aogeo/api/cda/caiji/status";
+ public static final String STATUS = "status";
+ /************************redis*************************************/
+ public static final String LOCK_KEY = "myLock";
+ public static final long LOCK_EXPIRE_TIME = 300000;
+
+ /************************应用参数*************************************/
+ public static final String CODE = "code";
+ public static final String MESSAGE = "message";
+ public static final String INPUT = "input";
+ public static final String OUTPUT = "output";
+ public static final String FORM = "form";
+ public static final String FIELD = "field";
+ public static final String VALUE = "value";
+ public static final String DATA = "data";
+ public static final String COLON_EN = ":";
+ /******************************admin*******************************/
+ public static final String AUTHORIZATION = "authorization";
+ public static final String TEMPERATURE = "temperature";
+ public static final String FIELDTYPE = "fieldType";
+ public static final String TOP_P = "top_p";
+ public static final String MODEL = "model";
+ public static final String PROMPT = "prompt";
+ public static final String CHATGLM_TURBO_URL = "https://open.bigmodel.cn/api/paas/v4/chat/completions";
+ public static final String CHOICES = "choices";
+ public static final String CONTENT = "content";
+}
diff --git a/src/main/java/com/bfd/qanda/handler/MainHandler.java b/src/main/java/com/bfd/qanda/handler/MainHandler.java
new file mode 100644
index 0000000..c4a0365
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/handler/MainHandler.java
@@ -0,0 +1,128 @@
+package com.bfd.qanda.handler;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import javax.annotation.Resource;
+
+import com.bfd.util.PauseTool;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.io.FileUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
+import org.springframework.core.annotation.Order;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Component;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.qanda.cache.ConfigCache;
+import com.bfd.qanda.process.QandaPorcess;
+import com.bfd.qanda.utils.FileUtil;
+
+
+/**
+ * 启动处理入口
+ *
+ * @author jian.mao
+ * @date 2023年11月3日
+ * @description
+ */
+
+@Component
+@Order(value = 1)
+@Slf4j
+public class MainHandler implements ApplicationRunner {
+
+ @Value("${task.task-queue-path}")
+ private String taskPath;
+ @Value("${task.thread-num}")
+ private Integer threadNum;
+ @Resource
+ QandaPorcess qandaPorcess;
+ @Resource
+ private StringRedisTemplate stringRedisTemplate;
+ @Value("${zookeeper.connection-string}")
+ private String connectionString;
+ @Value("${zookeeper.publish-node}")
+ private String nodePath;
+
+ @Override
+ public void run(ApplicationArguments args) throws Exception {
+ PauseTool pauseTool = new PauseTool();
+ pauseTool.initializeRedisCache(stringRedisTemplate);
+ pauseTool.setupZookeeperListener(connectionString, nodePath);
+ for (Integer i = 0; i < threadNum; i++) {
+ //开启问答处理线程
+ new Thread(qandaPorcess).start();
+ log.info("开启问答处理线程-----");
+ }
+
+ //停止处理
+ waitDown();
+ //启动加载缓存任务
+ readTask(taskPath, ConfigCache.taskQueue);
+ }
+
+
+ @SuppressWarnings("unchecked")
+ public static void readTask(String path, LinkedBlockingDeque> queue) {
+ File file = new File(path);
+ if (file.exists()) {
+ List tasks = null;
+ try {
+ tasks = FileUtils.readLines(file, "UTF-8");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ for (String taskStr : tasks) {
+ Map task = JSONObject.parseObject(taskStr);
+ try {
+ queue.put(task);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ file.delete();
+ }
+ }
+
+ /**
+ * 结束触发钩子
+ */
+ public void waitDown() {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ // 停止线程
+ ConfigCache.isStart = false;
+ log.info("stop-------");
+ writeTsskToFile();
+ }
+ });
+ }
+
+
+ /**
+ * 任务持久化到硬盘
+ */
+ public void writeTsskToFile() {
+ while (true) {
+ if (ConfigCache.taskQueue.size() > 0) {
+ try {
+ Map task = ConfigCache.taskQueue.take();
+ FileUtil.writeFile(taskPath, JSONObject.toJSONString(task));
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ } else {
+ log.info("taskQueue write is file end");
+ break;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/process/QandaPorcess.java b/src/main/java/com/bfd/qanda/process/QandaPorcess.java
new file mode 100644
index 0000000..f046dcc
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/process/QandaPorcess.java
@@ -0,0 +1,193 @@
+package com.bfd.qanda.process;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import com.bfd.qanda.utils.GPTResultParseUtil;
+import com.bfd.util.PauseTool;
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.qanda.cache.ConfigCache;
+import com.bfd.qanda.entity.Constants;
+import com.bfd.qanda.token.TokenManagerV3;
+import com.bfd.qanda.utils.DataUtil;
+import com.bfd.qanda.utils.DownLoadUtil;
+import com.bfd.qanda.utils.SpringBootKafka;
+
+/**
+ * @author jian.mao
+ * @date 2023年11月8日
+ * @description
+ */
+@Component
+@Slf4j
+public class QandaPorcess implements Runnable {
+ private static final int CHAT_TYPE_ONE = 1;
+ @Autowired
+ private SpringBootKafka springBootKafka;
+ @Value("${customize-kafka.producer.topic}")
+ private String topic;
+
+ @Override
+ public void run() {
+ while (ConfigCache.isStart) {
+ Map task = null;
+ try {
+ task = ConfigCache.taskQueue.take();
+ //结果接收
+ Map result = new HashMap(16);
+ //结果内容
+ StringBuffer chatContent = new StringBuffer();
+ //输入配置
+ Map input = (Map) task.get(Constants.INPUT);
+ //输出配置
+ Map output = (Map) task.get(Constants.OUTPUT);
+ //数据源
+ Map data = (Map) task.get(Constants.DATA);
+ int scenesId = (int) task.get(Constants.SCENES_ID);
+ int version = (int) task.get(Constants.VERSION);
+ String pauseKey = scenesId + "_" + version;
+ if (!PauseTool.CACHE.containsKey(pauseKey)) {
+ log.info("流程:{}的版本:{}已失效,任务跳过", scenesId, version);
+ continue;
+ }
+ //请求身份校验 api-key
+ String authorization = (String) input.get(Constants.AUTHORIZATION);
+ //fieldType：自定义输出字段： 0 关闭，1-开启，如果开启则拼接form到output里（如果关闭，则取默认的output拼接）
+ int fieldType = (int) input.get(Constants.FIELDTYPE);
+ Float temperature = Float.valueOf(input.get(Constants.TEMPERATURE).toString());
+ Float topP = Float.valueOf(input.get(Constants.TOP_P).toString());
+ String model = (String) input.get(Constants.MODEL).toString();
+ List> prompt = (List>) input.get(Constants.PROMPT);
+ String answerStr = qandaRequest(authorization, model, temperature, topP, prompt, data);
+ log.info("answerStr:" + answerStr);
+ Map answer = JSONObject.parseObject(answerStr);
+ try {
+ //请求成功，正常解析
+ List> choices = (List>) answer.get(Constants.CHOICES);
+ Map message = (Map) choices.get(0).get(Constants.MESSAGE);
+ chatContent.append(message.get(Constants.CONTENT));
+ } catch (Exception e) {
+ log.error("问答接口响应体异常:{}", answerStr, e);
+ // TODO: handle exception
+ continue;
+ }
+ Map results = new HashMap(16);
+ results.put(Constants.ID, UUID.randomUUID().toString());
+ results.put(Constants.CONTENT, chatContent.toString());
+ if (fieldType != 0) {
+ results.remove(Constants.CONTENT);
+ try {
+ //请求成功，正常解析
+ List> choices = (List>) answer.get(Constants.CHOICES);
+ Map message = (Map) choices.get(0).get(Constants.MESSAGE);
+ String reponseContent = (String) message.get(Constants.CONTENT);
+ Map stringObjectMap = GPTResultParseUtil.parseGPTResult(output, reponseContent);
+ results.putAll(stringObjectMap);
+ } catch (Exception e) {
+ log.error("问答接口响应体异常:{}", answerStr, e);
+ // TODO: handle exception
+ continue;
+ }
+ }
+ results.put("isLast", 1);
+ result.put(Constants.RESULTS, JSONObject.toJSONString(results));
+ result.put(Constants.MESSAGE, "成功");
+ result.put(Constants.STATUS, 1);
+ task.put(Constants.RESULT, result);
+// //发送kafka
+ springBootKafka.send(topic, JSONObject.toJSONString(task));
+ log.info("数据流转至下游-------");
+ } catch (Exception e) {
+ log.error("问答处理异常，", e);
+ //结果集
+ Map results = new HashMap(16);
+
+ Map result = new HashMap(16);
+ //遍历入库返回结果，拼接响应内容
+ results.put("isLast", 1);
+ results.put("content", e.getMessage());
+ result.put(Constants.RESULTS, JSONObject.toJSONString(results));
+ result.put(Constants.MESSAGE, "异常");
+ result.put(Constants.STATUS, 2);
+ task.put(Constants.RESULT, result);
+ //发送kafka
+ springBootKafka.send(topic, JSONObject.toJSONString(task));
+ log.info("数据流转至下游-------");
+
+
+ }
+
+ }
+ }
+
+ /**
+ * 问题请求
+ *
+ * @param authorization
+ * @param temperature
+ * @param topP
+ * @param prompt
+ * @return
+ */
+ private String qandaRequest(String authorization, String model, Float temperature, Float topP, List> prompt, Map data) {
+ //新建聊天话术
+ StringBuffer chatContent = new StringBuffer();
+ for (Map map : prompt) {
+ if (Integer.valueOf(map.get(Constants.TYPE).toString()) == CHAT_TYPE_ONE) {
+ chatContent.append(map.get(Constants.VALUE));
+ } else {
+ String jsonPath = (String) map.get(Constants.VALUE);
+ chatContent.append(DataUtil.getValue(jsonPath, data).toString());
+ }
+ }
+ //拆分authorization
+ String[] apiCombination = authorization.split("\\.");
+ //获取token
+ String token = TokenManagerV3.getToken(apiCombination[0], apiCombination[1]);
+ StringBuffer encryptionAuthorization = new StringBuffer("Bearer ");
+ encryptionAuthorization.append(token);
+ Map headers = new HashMap(16);
+ Map params = new HashMap(16);
+ List> prompts = buildParam(chatContent.toString(), data);
+
+ params.put("model", model);
+ params.put("messages", prompts);
+ params.put("temperature", temperature);
+ params.put("top_p", topP);
+
+ headers.put("Authorization", encryptionAuthorization.toString());
+ headers.put("Content-Type", "application/json");
+ String html = DownLoadUtil.doPost(Constants.CHATGLM_TURBO_URL, JSONObject.toJSONString(params), headers);
+ return html;
+ }
+
+ /**
+ * 参数构建
+ *
+ * @param prompt
+ * @param data
+ * @return
+ */
+ private List> buildParam(String prompt, Map data) {
+ //聊天体
+ List> prompts = new ArrayList>();
+ log.info("question---{}", prompt);
+ Map chat1 = new HashMap(16);
+ chat1.put("role", "user");
+ chat1.put("content", prompt);
+ prompts.add(chat1);
+ return prompts;
+ }
+
+
+}
diff --git a/src/main/java/com/bfd/qanda/service/QandaService.java b/src/main/java/com/bfd/qanda/service/QandaService.java
new file mode 100644
index 0000000..ce5f79e
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/service/QandaService.java
@@ -0,0 +1,17 @@
+package com.bfd.qanda.service;
+
+/**
+ * 问答业务层
+ * @author jian.mao
+ * @date 2023年11月8日
+ * @description
+ */
+public interface QandaService {
+
+ /**
+ * 问答
+ * @param dataJson 接口参数
+ * @return
+ */
+ public String putQuestion(String dataJson);
+}
diff --git a/src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java b/src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java
new file mode 100644
index 0000000..98a205b
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java
@@ -0,0 +1,58 @@
+package com.bfd.qanda.service.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.stereotype.Service;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.qanda.cache.ConfigCache;
+import com.bfd.qanda.entity.Constants;
+import com.bfd.qanda.service.QandaService;
+
+/**
+ * 业务层实现类
+ *
+ * @author jian.mao
+ * @date 2023年11月8日
+ * @description
+ */
+@Service
+@Slf4j
+public class QandaServiceImpl implements QandaService {
+
+ @Override
+ public String putQuestion(String dataJson) {
+ String trace = "trace";
+ Map response = new HashMap<>(16);
+ int code = 200;
+ String message = "success";
+ Map task = null;
+ try {
+ task = JSONObject.parseObject(dataJson);
+ } catch (Exception e) {
+ log.error("参数结构不合法，", e);
+ code = 100010;
+ message = "参数不合法";
+ }
+ // 写入队列
+ try {
+ if (task.containsKey(trace) && (Boolean) task.get(trace)==true) {
+ log.info("测试流程,插入队首");
+ ConfigCache.taskQueue.putFirst(task);
+ }else {
+ ConfigCache.taskQueue.put(task);
+ }
+ } catch (InterruptedException e) {
+ log.error("任务写入等待队列异常，", e);
+ code = 100011;
+ message = "任务写入等待队列失败";
+ }
+ response.put(Constants.CODE, code);
+ response.put(Constants.MESSAGE, message);
+ return JSONObject.toJSONString(response);
+ }
+
+}
diff --git a/src/main/java/com/bfd/qanda/token/TokenManagerV3.java b/src/main/java/com/bfd/qanda/token/TokenManagerV3.java
new file mode 100644
index 0000000..ebe9b89
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/token/TokenManagerV3.java
@@ -0,0 +1,93 @@
+package com.bfd.qanda.token;
+
+import java.util.Calendar;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import lombok.extern.slf4j.Slf4j;
+
+import com.auth0.jwt.JWT;
+import com.auth0.jwt.algorithms.Algorithm;
+import com.bfd.qanda.cache.LocalCache;
+import com.bfd.qanda.entity.Constants;
+
+/**
+ * token管理类
+ * @author jian.mao
+ * @date 2023年11月8日
+ * @description
+ */
+@Slf4j
+public class TokenManagerV3 {
+
+ private static final String TOKENV3KEYPREFIX = "zhipu_oapi_token_v3";
+ private static final String ALG = "HS256";
+ /***默认过期时间30分钟****/
+ private static final int EXPIREMILLIS = 30 * 60 * 1000;
+
+ /**
+ * token 获取方法
+ * @param apiKey 用户id
+ * @param apiSecret 生成key
+ * @return
+ */
+ public static String getToken(String apiKey,String apiSecret) {
+ String tokenCacheKey = genTokenCacheKey(apiKey);
+ String cacheToken = LocalCache.get(tokenCacheKey);
+ if(cacheToken != null && !cacheToken.equals(Constants.EMPTY)){
+ return cacheToken;
+ }
+ String newToken = createJwt(apiKey,apiSecret);
+ LocalCache.set(tokenCacheKey, newToken, EXPIREMILLIS, TimeUnit.MILLISECONDS);
+ return newToken;
+ }
+
+ /**
+ * token 计算
+ * @param apiKey
+ * @param apiSecret
+ * @return
+ */
+ private static String createJwt(String apiKey,String apiSecret) {
+ Algorithm alg;
+ String hs = "HS256";
+ if (ALG.equals(hs)) {
+ //alg = Algorithm.HMAC256(config.getApiSecret());
+ try {
+ alg = Algorithm.HMAC256(apiSecret.getBytes("utf-8"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ }
+ } else {
+ // 目前仅支持HS256
+ log.error("algorithm: %s not supported", ALG);
+ return null;
+ }
+
+ Map payload = new HashMap<>(16);
+ payload.put("api_key", apiKey);
+ payload.put("exp", System.currentTimeMillis() + EXPIREMILLIS);
+ payload.put("timestamp", Calendar.getInstance().getTimeInMillis());
+ Map headerClaims = new HashMap<>(16);
+ headerClaims.put("alg", "HS256");
+ headerClaims.put("sign_type", "SIGN");
+ String token = JWT.create().withPayload(payload).withHeader(headerClaims).sign(alg);
+ return token;
+ }
+
+ /**
+ * 获取缓存key
+ * @param apiKey
+ * @return
+ */
+ private static String genTokenCacheKey(String apiKey) {
+ return String.format("%s-%s", TOKENV3KEYPREFIX, apiKey);
+ }
+ public static void main(String[] args) {
+ String apiKey ="53dbef19a3252a292890408707190f9f";
+ String apiSecret="5An47w6PngrTFw19";
+ System.out.println(getToken(apiKey,apiSecret));
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/utils/DataUtil.java b/src/main/java/com/bfd/qanda/utils/DataUtil.java
new file mode 100644
index 0000000..6ca3666
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/DataUtil.java
@@ -0,0 +1,48 @@
+package com.bfd.qanda.utils;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.JSONPath;
+
+import java.util.Map;
+
+/**
+ * @author:jinming
+ * @className:DataUtil
+ * @version:1.0
+ * @description: 获取dataValue的值
+ * @Date:2023/11/1 9:54
+ */
+public class DataUtil {
+ /**
+ *
+ * @param key 传入的key
+ * @param dataMap 数据map
+ * @return 根据传入的参数进行判断解析，返回正确的dataValue
+ */
+ public static Object getValue(String key, Map dataMap) {
+ Object dataValue;
+ String isJson = "#json#";
+ if (key.contains(isJson)) {
+ //进行第一次拆分，获取#json#前面的部分
+ String[] keySplit = key.split(isJson);
+ String firstDataKey = keySplit[0];
+ String[] firstDataKeySplit = firstDataKey.split(":");
+ //取出前半部分对应的JSON数据并转换为JSONObject
+ String dataJson = (String) dataMap.get(firstDataKeySplit[0]);
+ JSONObject dataJsonObject = JSON.parseObject(dataJson);
+ //根据key的后半部分取出对应JSONObject中的值
+ String firstDataKeyJson = (String) JSONPath.eval(dataJsonObject, firstDataKeySplit[1]);
+ String secDataKey = keySplit[1];
+ JSONObject firstDataJsonObject = JSON.parseObject(firstDataKeyJson);
+ dataValue = JSONPath.eval(firstDataJsonObject, secDataKey);
+ return dataValue;
+ }
+ String[] keySplit = key.split(":");
+ String jsonPath = keySplit[1];
+ String dataJson = (String) dataMap.get(keySplit[0]);
+ JSONObject dataJsonObject = JSON.parseObject(dataJson);
+ dataValue = JSONPath.eval(dataJsonObject, jsonPath);
+ return dataValue;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/qanda/utils/DateUtil.java b/src/main/java/com/bfd/qanda/utils/DateUtil.java
new file mode 100644
index 0000000..0b42b01
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/DateUtil.java
@@ -0,0 +1,177 @@
+package com.bfd.qanda.utils;
+
+
+import java.math.BigInteger;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+
+import lombok.extern.slf4j.Slf4j;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+
+/**
+ * 日期工具类
+ *
+ * @author jian.mao
+ * @date 2022年11月15日
+ * @description
+ */
+@Slf4j
+public class DateUtil {
+
+ /**
+ * @return
+ */
+ public static String getTimeStrForNow() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH");
+ return sdf.format(new Date());
+ }
+
+
+ public static String getTimeStrForDay(long time) {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+
+ return sdf.format(new Date(time * 1000));
+ }
+
+ public static String getTimeStrForDay() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+
+ return sdf.format(new Date());
+ }
+
+
+ public static String getDateTime() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String time = sdf.format(new Date());
+ return time;
+ }
+
+ public static String getDateTime(Long timestap) {
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String time = sdf.format(new Date(timestap));
+ return time;
+ }
+
+ public static String getDate(Long timestap) {
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+ String time = sdf.format(new Date(timestap));
+ return time;
+ }
+
+ public static String getDateTimeForMonth() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM");
+ String time = sdf.format(new Date());
+ return time;
+ }
+
+ /**
+ * 休眠
+ *
+ * @param millis 毫秒
+ */
+ public static void sleep(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * 1. @Description:时间戳转时间
+ * 2. @Author: ying.zhao
+ * 3. @Date: 2023/3/28
+ */
+
+ public static String timestampToDate(String time) {
+ int thirteen = 13;
+ int ten = 10;
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+// if (time.length() == thirteen) {
+ if (time.length() > ten) {
+ return sdf.format(new Date(Long.parseLong(time)));
+ } else {
+ return sdf.format(new Date(Integer.parseInt(time) * 1000L));
+ }
+ }
+
+ public static String parseCreated(String jsonTime){
+ String formattedDateTime = getDateTime();
+ try {
+ // 使用fastjson解析JSON数据
+ JSONObject jsonObject = JSON.parseObject(jsonTime);
+ // 获取日期和时间的值
+ JSONObject dateObject = jsonObject.getJSONObject("date");
+ int day = dateObject.getIntValue("day");
+ int month = dateObject.getIntValue("month");
+ int year = dateObject.getIntValue("year");
+
+ JSONObject timeObject = jsonObject.getJSONObject("time");
+ int hour = timeObject.getIntValue("hour");
+ int minute = timeObject.getIntValue("minute");
+ int second = timeObject.getIntValue("second");
+
+ // 创建LocalDateTime对象
+ LocalDateTime dateTime = LocalDateTime.of(year, month, day, hour, minute, second);
+
+ // 定义日期时间格式化器
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+ // 格式化日期时间
+ formattedDateTime = dateTime.format(formatter);
+ } catch (Exception e) {
+ log.info("日期转换失败：{}",e);
+ }
+ return formattedDateTime;
+ }
+
+ /**
+ * 字符串转换日期
+ * @param format
+ * @param date
+ * @return
+ */
+ public static Date strToDate(String format,String date){
+ SimpleDateFormat sdf = new SimpleDateFormat(format);
+ if (date == null || date.equals("")){
+ return new Date();
+ }else{
+ Date ru = null;
+ try {
+ ru = sdf.parse(date);
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+ return ru;
+ }
+ }
+ /**
+ * 日期格式话
+ * @param format 日期格式
+ * @param dater 要转换的日期,默认当前时间
+ * @return
+ */
+ public static String FormatDate(String format,Date date){
+ String fromatDate = null;
+ SimpleDateFormat sdf = new SimpleDateFormat(format);
+ if (date == null){
+ fromatDate = sdf.format(new Date());
+ }else{
+ fromatDate = sdf.format(date);
+ }
+ return fromatDate;
+ }
+ public static void main(String[] args) {
+ String time = timestampToDate("955814400000");
+ System.out.println(time);
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/utils/DownLoadUtil.java b/src/main/java/com/bfd/qanda/utils/DownLoadUtil.java
new file mode 100644
index 0000000..9f164b2
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/DownLoadUtil.java
@@ -0,0 +1,914 @@
+package com.bfd.qanda.utils;
+
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.StatusLine;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.AuthCache;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.config.SocketConfig;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.LaxRedirectStrategy;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.bfd.qanda.entity.Constants;
+
+
+
+
+
+
+
+/**
+ * 下载工具类
+ * @author jian.mao
+ * @date 2023年9月19日
+ * @description
+ */
+public class DownLoadUtil {
+
+ private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36";
+ private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class);
+ /** 代理服务器(产品官网 www.16yun.cn) **/
+ final static String PROXYHOST = "u270.40.tp.16yun.cn";
+ final static Integer PROXYPORT = 6448;
+ /** 代理验证信息 **/
+ final static String PROXYUSER = "16HFBVJC";
+ final static String PROXYPASS = "897944";
+
+ private static PoolingHttpClientConnectionManager cm = null;
+ private static HttpRequestRetryHandler httpRequestRetryHandler = null;
+ private static HttpHost proxy = null;
+
+ private static CredentialsProvider credsProvider = null;
+ private static RequestConfig reqConfig = null;
+
+ static {
+ ConnectionSocketFactory plainsf = PlainConnectionSocketFactory
+ .getSocketFactory();
+ LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory
+ .getSocketFactory();
+
+ Registry registry = RegistryBuilder.create().register("http", plainsf)
+ .register("https", sslsf).build();
+
+ cm = new PoolingHttpClientConnectionManager(registry);
+ cm.setMaxTotal(20);
+ cm.setDefaultMaxPerRoute(5);
+
+ proxy = new HttpHost(PROXYHOST, PROXYPORT, "https");
+
+ credsProvider = new BasicCredentialsProvider();
+ credsProvider.setCredentials(AuthScope.ANY,
+ new UsernamePasswordCredentials(PROXYUSER, PROXYPASS));
+
+ reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000)
+ .setConnectTimeout(5000).setSocketTimeout(5000)
+ .setExpectContinueEnabled(false)
+ .setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build();
+ }
+
+ /**
+ * 模拟客户端get请求
+ *
+ * @param url
+ * 模拟请求得url
+ * @param headers
+ * 头部信息，没有可以不传
+ * @return
+ */
+ @SafeVarargs
+ public static String proxyDoGet(String url, Map... headers) {
+ // 设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom()
+ .setConnectTimeout(timeout * 1000)
+ .setConnectionRequestTimeout(timeout * 1000)
+ .setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(timeout * 1000)
+ .setTcpNoDelay(true).build();
+ AuthCache authCache = new BasicAuthCache();
+ authCache.put(proxy, new BasicScheme());
+ HttpClientContext localContext = HttpClientContext.create();
+ localContext.setAuthCache(authCache);
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ CloseableHttpClient httpClient = httpBuilder
+ .setDefaultSocketConfig(socketConfig)
+ .setDefaultRequestConfig(config)
+ .setDefaultCredentialsProvider(credsProvider).build();
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setConfig(reqConfig);
+ if (headers != null && headers.length > 0) {
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key, tempHeaders.get(key).toString());
+ }
+ } else {
+ httpGet.setHeader("Accept",
+ "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
+ }
+ CloseableHttpResponse response = null;
+ String html = "";
+ int notFundCode = 404;
+ int successCode = 200;
+ try {
+ response = httpClient.execute(httpGet, localContext);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if (statusLine.getStatusCode() == successCode) {
+ if (responseEntity != null) {
+ html = EntityUtils.toString(responseEntity, "utf-8");
+ System.out.println("响应内容长度为:"
+ + responseEntity.getContentLength());
+ // 下载结果为空不正常
+ if (html.equals(Constants.EMPTY)) {
+ html = "Download failed error is:reslut is null";
+ }
+ }
+ } else if (statusLine.getStatusCode() == notFundCode) {
+ html = "页面404，正常结束请求即可
";
+ } else {
+ throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:reslut is null";
+ }finally{
+ try {
+ response.close();
+ httpClient.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ return html;
+
+ }
+
+
+ public static String httpsslProxyGet(String url, Map... headers) throws Exception {
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connManager.setMaxTotal(50);
+ connManager.setDefaultMaxPerRoute(10);
+ HttpClients.custom().setConnectionManager(connManager);
+ // 设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom()
+ .setConnectTimeout(timeout * 1000)
+ .setConnectionRequestTimeout(timeout * 1000)
+ .setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(timeout * 1000)
+ .setTcpNoDelay(true).build();
+ AuthCache authCache = new BasicAuthCache();
+ authCache.put(proxy, new BasicScheme());
+ HttpClientContext localContext = HttpClientContext.create();
+ localContext.setAuthCache(authCache);
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ CloseableHttpClient httpClient = httpBuilder
+ .setConnectionManager(connManager)
+ .setDefaultSocketConfig(socketConfig)
+ .setDefaultRequestConfig(config)
+ .setDefaultCredentialsProvider(credsProvider).build();
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setConfig(reqConfig);
+ if (headers != null && headers.length > 0) {
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key, tempHeaders.get(key).toString());
+ }
+ } else {
+ httpGet.setHeader("Accept",
+ "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
+ }
+ CloseableHttpResponse response = null;
+ String html = "";
+ int notFundCode = 404;
+ int successCode = 200;
+ try {
+ response = httpClient.execute(httpGet, localContext);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if (statusLine.getStatusCode() == successCode) {
+ if (responseEntity != null) {
+ html = EntityUtils.toString(responseEntity, "utf-8");
+ System.out.println("响应内容长度为:"
+ + responseEntity.getContentLength());
+ // 下载结果为空不正常
+ if (html.equals(Constants.EMPTY)) {
+ html = "Download failed error is:reslut is null";
+ }
+ }
+ } else if (statusLine.getStatusCode() == notFundCode) {
+ html = "页面404，正常结束请求即可
";
+ } else {
+ throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:reslut is null";
+ }finally{
+ try {
+ response.close();
+ httpClient.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ return html;
+
+ }
+
+
+ /**
+ * json参数方式POST提交
+ * @param url
+ * @param params
+ * @return
+ */
+ public static String doPost(String url, String params, Map... headers){
+ String strResult = "";
+ //设置超时时间
+ int timeout = 60;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(timeout * 1000)
+ .setTcpNoDelay(true).build();
+// AuthCache authCache = new BasicAuthCache();
+// authCache.put(proxy, new BasicScheme());
+// HttpClientContext localContext = HttpClientContext.create();
+// localContext.setAuthCache(authCache);
+ // 1. 获取默认的client实例
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ httpBuilder.setUserAgent(ua);
+ HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
+// HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
+// .setDefaultCredentialsProvider(credsProvider).build();
+ // 2. 创建httppost实例
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ if (headers != null && headers.length > 0) {
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpPost.setHeader(key, tempHeaders.get(key).toString());
+ }
+ } else {
+ httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
+ }
+ HttpResponse resp = null;
+ try {
+ httpPost.setEntity(new StringEntity(params,"utf-8"));
+ resp = client.execute(httpPost);
+// resp = client.execute(httpPost,localContext);
+ StatusLine statusLine = resp.getStatusLine();
+ System.out.println("响应状态为:" + resp.getStatusLine());
+ int notFundCode = 404;
+ int successCode = 200;
+ if(statusLine.getStatusCode() == successCode){
+ // 7. 获取响应entity
+ HttpEntity respEntity = resp.getEntity();
+ strResult = EntityUtils.toString(respEntity, "UTF-8");
+ if(strResult.equals(Constants.EMPTY)){
+ strResult = "Download failed error is:reslut is null";
+ }
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+ return strResult;
+ }
+ public static String httpPost(String url,String params) {
+ String html="";
+ html = doPost(url,params);
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(5000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ html = doPost(url,params);
+ }
+ return html;
+ }
+ /**
+ * 绕过验证
+ *
+ * @return
+ * @throws NoSuchAlgorithmException
+ * @throws KeyManagementException
+ */
+ public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
+ SSLContext sc = SSLContext.getInstance("SSLv3");
+
+ // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
+ X509TrustManager trustManager = new X509TrustManager() {
+ @Override
+ public void checkClientTrusted(
+ java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
+ String paramString) throws CertificateException {
+ }
+
+ @Override
+ public void checkServerTrusted(
+ java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
+ String paramString) throws CertificateException {
+ }
+
+ @Override
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ };
+
+ sc.init(null, new TrustManager[] { trustManager }, null);
+ return sc;
+ }
+ /**
+ * 模拟请求
+ *
+ * @param url 资源地址
+ * @param map 参数列表
+ * @param encoding 编码
+ * @return
+ * @throws NoSuchAlgorithmException
+ * @throws KeyManagementException
+ * @throws IOException
+ * @throws ClientProtocolException
+ */
+ public static String httpsslGet(String url,Map ... headers) {
+ String html="";
+ CloseableHttpClient client = null;
+ HttpEntity responseEntity = null;
+ CloseableHttpResponse response = null;
+ try {
+ log.debug("DownLoadUtil------------->设置下载相关信息， start....");
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connManager.setMaxTotal(50);
+ connManager.setDefaultMaxPerRoute(10);
+ HttpClients.custom().setConnectionManager(connManager);
+ //设置超时时间
+ int timeout = 30;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ // 设置重定向策略
+ LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy();
+ //创建自定义的httpclient对象
+ client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build();
+// CloseableHttpClient client = HttpClients.createDefault();
+
+ HttpGet httpGet = new HttpGet(url);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
+ }
+ log.debug("DownLoadUtil------------->设置下载相关信息， end....");
+ try {
+ int notFundCode = 404;
+ int successCode = 200;
+ log.debug("DownLoadUtil------------->下载执行，start....");
+ httpGet.setConfig(config);
+ response = client.execute(httpGet);
+ log.debug("DownLoadUtil------------->下载执行，end....");
+ // 从响应模型中获取响应实体
+ StatusLine statusLine = response.getStatusLine();
+ log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is start ....");
+ System.out.println("响应状态为:" + response.getStatusLine());
+ responseEntity = response.getEntity();
+ log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is end ....");
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ System.out.println("响应内容长度为:" + responseEntity.getContentLength());
+ }
+ }else if(statusLine.getStatusCode() == notFundCode){
+ html = "页面404，正常结束请求即可
";
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }finally{
+ try {
+ responseEntity.getContent().close();
+ response.close();
+ client.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
+ return html;
+ }
+
+ public static String httpSSLGet(String url,Map ... headers) {
+ String html="";
+ html = httpsslGet(url,headers);
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(30000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ html = httpsslGet(url,headers);
+ }
+ return html;
+ }
+ public static String doPostFrom(String url,Map param,Map ... headers){
+ //设置超时时间
+ int timeout = 15;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+// AuthCache authCache = new BasicAuthCache();
+// authCache.put(proxy, new BasicScheme());
+// HttpClientContext localContext = HttpClientContext.create();
+// localContext.setAuthCache(authCache);
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ httpBuilder.setUserAgent(ua);
+// HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
+// .setDefaultCredentialsProvider(credsProvider).build();
+ HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpPost.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
+ httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
+ httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
+ httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36");
+// httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html");
+ }
+ // 创建请求参数
+ List list = new LinkedList<>();
+ for (String key : param.keySet()) {
+ BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
+ list.add(param1);
+ }
+ // 使用URL实体转换工具
+ String html="";
+ try {
+ UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
+ httpPost.setEntity(entityParam);
+ HttpResponse response = httpClient.execute(httpPost);
+// HttpResponse response = httpClient.execute(httpPost,localContext);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ int notFundCode = 404;
+ int successCode = 200;
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ }
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+
+ return html;
+
+ }
+ public static String httpPostForm(String url,Map params,Map ... headers) {
+ String html="";
+ html = doPostFrom(url,params);
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(5000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ html = doPostFrom(url,params,headers);
+ }
+ return html;
+ }
+
+ public static String dosslPost(String url,String params,Map ... headers) {
+ String html="";
+ CloseableHttpClient client = null;
+ HttpEntity responseEntity = null;
+ CloseableHttpResponse response = null;
+ try {
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ HttpClients.custom().setConnectionManager(connManager);
+ //设置超时时间
+ int timeout = 5;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ //创建自定义的httpclient对象
+ client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
+// CloseableHttpClient client = HttpClients.createDefault();
+ // 2. 创建httppost实例
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpPost.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
+ }
+
+ try {
+ httpPost.setEntity(new StringEntity(params,"utf-8"));
+ response = client.execute(httpPost);
+ int notFundCode = 404;
+ int successCode = 200;
+ // 从响应模型中获取响应实体
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ responseEntity = response.getEntity();
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ System.out.println("响应内容长度为:" + responseEntity.getContentLength());
+ }
+ }else if(statusLine.getStatusCode() == notFundCode){
+ html = "页面404，正常结束请求即可
";
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }finally{
+ try {
+ responseEntity.getContent().close();
+ response.close();
+ client.close();
+ } catch (UnsupportedOperationException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
+ return html;
+ }
+ public static String dosslPostForm(String url,Map param,Map ... headers) {
+ String html="";
+ try {
+ //采用绕过验证的方式处理https请求
+ SSLContext sslcontext = createIgnoreVerifySSL();
+
+ // 设置协议http和https对应的处理socket链接工厂的对象
+ Registry socketFactoryRegistry = RegistryBuilder.create()
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .register("https", new SSLConnectionSocketFactory(sslcontext))
+ .build();
+ PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ HttpClients.custom().setConnectionManager(connManager);
+ //设置超时时间
+ int timeout = 5;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ //创建自定义的httpclient对象
+ CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
+// CloseableHttpClient client = HttpClients.createDefault();
+ // 2. 创建httppost实例
+ HttpPost httpPost = new HttpPost(url);
+// httpPost.setConfig(reqConfig);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpPost.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
+ httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
+ }
+
+ // 创建请求参数
+ List list = new LinkedList<>();
+ for (String key : param.keySet()) {
+ BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
+ list.add(param1);
+ }
+ // 使用URL实体转换工具
+ try {
+ UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
+ httpPost.setEntity(entityParam);
+ HttpResponse response = client.execute(httpPost);
+// HttpResponse response = httpClient.execute(httpPost,localContext);
+ // 从响应模型中获取响应实体
+ int notFundCode = 404;
+ int successCode = 200;
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ }
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+
+
+ return html;
+ }
+ public static String httpSSLPostForm(String url,Map params,Map ...headers) {
+ String html="";
+ try {
+ html = dosslPostForm(url,params,headers);
+ } catch (Exception e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(30000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ try {
+ html = dosslPostForm(url,params,headers);
+ } catch (Exception e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ }
+ return html;
+ }
+ public static String httpSSLPost(String url,String params,Map ...headers) {
+ String html="";
+ try {
+ html = dosslPost(url,params,headers);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ int i = 1;
+ while(true){
+ if(html.contains("Download failed error is:")){
+ log.error("DownLoadUtil------------->download is failure,url is:"+url);
+ DateUtil.sleep(30000);
+ i++;
+ }else{
+ break;
+ }
+ if(i > 5){
+ break;
+ }
+ try {
+ html = dosslPost(url,params,headers);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ // TODO: handle exception
+ html = "Download failed error is:Exception!";
+ }
+ }
+ return html;
+ }
+
+ /**
+ * 模拟客户端get请求
+ * @param url 模拟请求得url
+ * @param headers 头部信息，没有可以不传
+ * @return
+ */
+ public static String doGet(String url,Map ... headers){
+ //设置超时时间
+ int timeout = 15;
+ RequestConfig config = RequestConfig.custom().
+ setConnectTimeout(timeout * 1000).
+ setConnectionRequestTimeout(timeout * 1000).
+ setSocketTimeout(timeout * 1000).build();
+ SocketConfig socketConfig = SocketConfig.custom()
+ .setSoKeepAlive(false)
+ .setSoLinger(1)
+ .setSoReuseAddress(true)
+ .setSoTimeout(10000)
+ .setTcpNoDelay(true).build();
+ HttpClientBuilder httpBuilder = HttpClientBuilder.create();
+ httpBuilder.setUserAgent(ua);
+ HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
+ HttpGet httpGet = new HttpGet(url);
+ if(headers != null && headers.length > 0){
+ Map tempHeaders = headers[0];
+ for (String key : tempHeaders.keySet()) {
+ httpGet.setHeader(key,tempHeaders.get(key).toString());
+ }
+ }else{
+ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+ httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
+ }
+ String html="";
+ try {
+ int notFundCode = 404;
+ int successCode = 200;
+ HttpResponse response = httpClient.execute(httpGet);
+ // 从响应模型中获取响应实体
+ HttpEntity responseEntity = response.getEntity();
+ StatusLine statusLine = response.getStatusLine();
+ System.out.println("响应状态为:" + response.getStatusLine());
+ if(statusLine.getStatusCode() == successCode){
+ if (responseEntity != null) {
+ html=EntityUtils.toString(responseEntity,"utf-8");
+ if(html.equals("")){
+ html = "Download failed error is:reslut is null";
+ }
+ }
+ }else if(statusLine.getStatusCode() == notFundCode){
+ html = "页面404，正常结束请求即可
";
+ }else{
+ throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
+ }
+ return html;
+
+ }
+ public static void main(String[] args) throws Exception {
+
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/utils/EncryptionUtil.java b/src/main/java/com/bfd/qanda/utils/EncryptionUtil.java
new file mode 100644
index 0000000..1efdef6
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/EncryptionUtil.java
@@ -0,0 +1,27 @@
+package com.bfd.qanda.utils;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * @author jian.mao
+ * @date 2023年3月10日
+ * @description
+ */
+public class EncryptionUtil {
+ public static String md5(String text) {
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update(text.getBytes());
+ byte[] bytes = md.digest();
+ StringBuilder sb = new StringBuilder();
+ for (byte b : bytes) {
+ sb.append(String.format("%02x", b & 0xff));
+ }
+ return sb.toString();
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/utils/FileUtil.java b/src/main/java/com/bfd/qanda/utils/FileUtil.java
new file mode 100644
index 0000000..7e40c19
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/FileUtil.java
@@ -0,0 +1,36 @@
+package com.bfd.qanda.utils;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * 文件工具类
+ * @author jian.mao
+ * @date 2023年7月14日
+ * @description
+ */
+public class FileUtil {
+
+ /**
+ * 数据写入文件
+ * @param Path 文件路径
+ * @param result 数据
+ * @throws IOException
+ */
+ public static void writeFile(String path,String result){
+ try {
+ FileWriter fw = new FileWriter(path,true);
+ fw.write(result+"\n");
+ fw.flush();
+ fw.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java b/src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java
new file mode 100644
index 0000000..3bd86df
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java
@@ -0,0 +1,53 @@
+package com.bfd.qanda.utils;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.JSONException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author:jinming
+ * @className:GPTResultParseUtil
+ * @version:1.0
+ * @description:
+ * @Date:2024/6/28 10:11
+ */
+public class GPTResultParseUtil {
+ public static Map parseGPTResult(Map output, String gptContent) {
+ Map jsonResult = new HashMap<>();
+ try {
+ // 替换```json, ``` 和 \n
+ String jsonContent = gptContent.replace("```json", "").replace("```", "").replace("\n", "");
+ JSONObject jsonGPT = JSON.parseObject(jsonContent);
+
+ for (String key : output.keySet()) {
+ if (jsonGPT.containsKey(key)) {
+ jsonResult.put(key, jsonGPT.get(key));
+ }
+ }
+ return jsonResult;
+ } catch (JSONException e) {
+ try {
+ // 直接解析失败，使用正则表达式匹配外层的 {}
+ Pattern pattern = Pattern.compile("\\{.*\\}", Pattern.DOTALL);
+ Matcher matcher = pattern.matcher(gptContent.replace("\n", ""));
+ if (matcher.find()) {
+ JSONObject jsonGPT = JSON.parseObject(matcher.group());
+ for (String key : output.keySet()) {
+ if (jsonGPT.containsKey(key)) {
+ jsonResult.put(key, jsonGPT.get(key));
+ }
+ }
+ return jsonResult;
+ } else {
+ return null;
+ }
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ return null;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/qanda/utils/JsonUtil.java b/src/main/java/com/bfd/qanda/utils/JsonUtil.java
new file mode 100644
index 0000000..46fb9fb
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/JsonUtil.java
@@ -0,0 +1,32 @@
+package com.bfd.qanda.utils;
+
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.qanda.entity.Constants;
+
+/**
+ * json工具
+ * @author jian.mao
+ * @date 2023年7月10日
+ * @description
+ */
+public class JsonUtil {
+
+ /**
+ * 校验字符串是list/map/str
+ * @param jsonString
+ * @return
+ */
+ public static String checkJsonType(String jsonString) {
+ try {
+ JSONObject.parseObject(jsonString);
+ return Constants.MAP_TYPE;
+ } catch (Exception e) {
+ try {
+ JSONObject.parseArray(jsonString);
+ return Constants.LIST_TYPE;
+ } catch (Exception ex) {
+ return Constants.STRING_TYPE;
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/utils/OtherUtils.java b/src/main/java/com/bfd/qanda/utils/OtherUtils.java
new file mode 100644
index 0000000..0d8d9b2
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/OtherUtils.java
@@ -0,0 +1,33 @@
+package com.bfd.qanda.utils;
+
+import java.security.MessageDigest;
+
+/**
+ * 其他工具类
+ * @author jian.mao
+ * @date 2023年9月19日
+ * @description
+ */
+public class OtherUtils {
+
+
+
+ public static String getMd5(String string) {
+ try {
+ MessageDigest md5 = MessageDigest.getInstance("MD5");
+ byte[] bs = md5.digest(string.getBytes("UTF-8"));
+ StringBuilder sb = new StringBuilder(40);
+ for (byte x : bs) {
+ if ((x & 0xff) >> 4 == 0) {
+ sb.append("0").append(Integer.toHexString(x & 0xff));
+ } else {
+ sb.append(Integer.toHexString(x & 0xff));
+ }
+ }
+ return sb.toString();
+ } catch (Exception e) {
+
+ return "nceaform" + System.currentTimeMillis();
+ }
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/utils/QueueUtil.java b/src/main/java/com/bfd/qanda/utils/QueueUtil.java
new file mode 100644
index 0000000..5796c2a
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/QueueUtil.java
@@ -0,0 +1,18 @@
+package com.bfd.qanda.utils;
+
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingDeque;
+
+/**
+ * @author:jinming
+ * @className:QueueUtil
+ * @version:1.0
+ * @description:
+ * @Date:2023/7/13 15:00
+ */
+public class QueueUtil {
+
+ public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>();
+
+ public static LinkedBlockingDeque sendQueue = new LinkedBlockingDeque();
+}
\ No newline at end of file
diff --git a/src/main/java/com/bfd/qanda/utils/SpringBootKafka.java b/src/main/java/com/bfd/qanda/utils/SpringBootKafka.java
new file mode 100644
index 0000000..be8ad73
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/SpringBootKafka.java
@@ -0,0 +1,46 @@
+package com.bfd.qanda.utils;
+
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.SendResult;
+import org.springframework.stereotype.Component;
+import org.springframework.util.concurrent.ListenableFuture;
+import org.springframework.util.concurrent.ListenableFutureCallback;
+
+/**
+ * @PROJECT_NAME: companybusinesscrawl
+ * @DESCRIPTION:SpringBootKafka 工具类
+ * @AUTHOR: ying.zhao
+ * @DATE: 2023/4/6 11:09
+ */
+@Slf4j
+@Component
+public class SpringBootKafka {
+ @Autowired
+ private KafkaTemplate kafkaTemplate;
+ /**
+ * 自定义topicKafkaTemplate
+ */
+ /**
+ * public static final String TOPIC = "companyBussTest";
+ **/
+ public void send(String topic, String message) {
+ //发送消息
+ ListenableFuture> future = kafkaTemplate.send(topic, message);
+ future.addCallback(new ListenableFutureCallback>() {
+ @Override
+ public void onFailure(Throwable throwable) {
+ //发送失败的处理
+ log.info(topic + " - 生产者 发送消息失败：" + throwable.getMessage());
+ }
+
+ @Override
+ public void onSuccess(SendResult stringObjectSendResult) {
+ //成功的处理
+ log.info(topic + " - 生产者 发送消息成功" );
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java b/src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java
new file mode 100644
index 0000000..8771354
--- /dev/null
+++ b/src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java
@@ -0,0 +1,23 @@
+package com.bfd.qanda.utils;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+/**
+ * @author jian.mao
+ * @date 2023年3月22日
+ * @description
+ */
+public class ThrowMessageUtil {
+
+ /**
+ * 获取异常信息
+ * @param t
+ * @return
+ */
+ public static String getErrmessage(Throwable t){
+ StringWriter stringWriter=new StringWriter();
+ t.printStackTrace(new PrintWriter(stringWriter,true));
+ return stringWriter.getBuffer().toString();
+ }
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
new file mode 100644
index 0000000..a651fa6
--- /dev/null
+++ b/src/main/resources/application.yml
@@ -0,0 +1,85 @@
+logging:
+ level:
+ root: info
+ path: ./logs
+server:
+ port: 8010
+ servlet:
+ context-path: /bigmodel
+ tomcat:
+ uri-encoding: utf-8
+ max-connections: 20000
+ max-http-form-post-size: 1
+ max-threads: 1000
+spring:
+ application:
+ name: bigmodel
+ kafka:
+ bootstrap-servers: 172.26.28.30:9092
+ producer:
+ retries: 0
+ #当有多个消息需要被发送到同一个分区时，生产者会把它们放在同一个批次里。该参数指定了一个批次可以使用的内存大小，按照字节数计算。
+ batch-size: 16384
+ # 设置生产者内存缓冲区的大小。
+ buffer-memory: 33554432
+ # 键的序列化方式
+ key-serializer: org.apache.kafka.common.serialization.StringSerializer
+ # 值的序列化方式
+ value-serializer: org.apache.kafka.common.serialization.StringSerializer
+ # acks=0 ： 生产者在成功写入消息之前不会等待任何来自服务器的响应。
+ # acks=1 ： 只要集群的首领节点收到消息，生产者就会收到一个来自服务器成功响应。
+ # acks=all ：只有当所有参与复制的节点全部收到消息时，生产者才会收到一个来自服务器的成功响应。
+ acks: 1
+ consumer:
+ # 自动提交的时间间隔 在spring boot 2.X 版本中这里采用的是值的类型为Duration 需要符合特定的格式，如1S,1M,2H,5D
+ auto-commit-interval: 1S
+ # 该属性指定了消费者在读取一个没有偏移量的分区或者偏移量无效的情况下该作何处理：
+ # latest（默认值）在偏移量无效的情况下，消费者将从最新的记录开始读取数据（在消费者启动之后生成的记录）
+ # earliest ：在偏移量无效的情况下，消费者将从起始位置读取分区的记录
+ auto-offset-reset: earliest
+ # 是否自动提交偏移量，默认值是true,为了避免出现重复数据和数据丢失，可以把它设置为false,然后手动提交偏移量
+ enable-auto-commit: true
+ # 键的反序列化方式
+ key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
+ # 值的反序列化方式
+ value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
+ #消费组
+ group-id: test4
+ #消费者并发线程数
+ concurrency: 4
+ #超时时间
+ max-poll-interval-ms: 60000
+ #listener:
+ # 在侦听器容器中运行的线程数。
+ #concurrency: 5
+ #listner负责ack，每调用一次，就立即commit
+ #ack-mode: manual_immediate
+ #missing-topics-fatal: false
+
+ boot:
+ admin:
+ client:
+ url: http://172.16.12.55:8001
+ instance:
+ service-base-url: http://10.10.143.85:8010
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: "*"
+ endpoint:
+ health:
+ show-details: always
+ health:
+ elasticsearch:
+ enabled: false
+
+customize-kafka:
+ bootstrap-servers: 172.18.1.119:9992
+ producer:
+ topic: analyze
+task:
+ task-queue-path: ../data/taskQueue.txt
+ thread-num: 5
+
diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml
new file mode 100644
index 0000000..549c5ae
--- /dev/null
+++ b/src/main/resources/logback-spring.xml
@@ -0,0 +1,36 @@
+
+
+
+
+
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n
+
+
+
+
+ true
+
+ ${logging.level}
+
+
+ ${logging.path}/bigmodelInfo.log
+
+
+ ${logging.path}/bigmodelInfo.log.%d{yyyy-MM-dd}
+ 7
+
+
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n
+ UTF-8
+
+
+
+
+
+
+
+
diff --git a/src/test/java/com/bfd/AppTest.java b/src/test/java/com/bfd/AppTest.java
new file mode 100644
index 0000000..77a1a87
--- /dev/null
+++ b/src/test/java/com/bfd/AppTest.java
@@ -0,0 +1,53 @@
+package com.bfd;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.bfd.qanda.token.TokenManagerV3;
+import com.bfd.qanda.utils.DownLoadUtil;
+
+
+/**
+ * @author jian.mao
+ * @date 2023年11月13日
+ * @description
+ */
+public class AppTest {
+ public static void main(String[] args) {
+// String token = TokenManagerV3.getToken("46d834d33699ab5cc2cba4d1f9ffa96d", "XqIB1P3mxTMYGa5D");
+ String token = TokenManagerV3.getToken("9b1539749531dfc4d7615f51df700875", "HP0azw6gD0yIsbvt");
+// String url = "https://open.bigmodel.cn/api/paas/v3/model-api/chatglm_turbo/invoke";
+// String url = "https://open.bigmodel.cn/api/paas/v4/chat/completions";
+ String url = "https://open.bigmodel.cn/api/paas/v4/chat/completions";
+ Map headers = new HashMap(16);
+ Map params = new HashMap(16);
+ List> prompt = new ArrayList>();
+ Map chat1 = new HashMap(16);
+ String chatContent = "从下文中提取带有数字、产能等内容的关键句，没有相关内同请返回null。文本如下：共同富裕是社会主义的本质要求，是中国式现代化的重要特征。习近平总书记强调，“随着我国全面建成小康社会、开启全面建设社会主义现代化国家新征程，我们必须把促进全体人民共同富裕摆在更加重要的位置，脚踏实地，久久为功，向着这个目标更加积极有为地进行努力”。习近平总书记关于共同富裕的一系列重要论述，丰富和发展了马克思主义理论，丰富和发展了新时代中国共产党执政理念和治国方略，是对中国共产党人带领全体人民实现共同富裕奋斗目标的当代探索与实践，为我国实现第二个百年奋斗目标指明了方向、提供了遵循。";
+ chat1.put("role", "user");
+ chat1.put("content", chatContent);
+
+ prompt.add(chat1);
+ params.put("model", "glm-3-turbo");
+ params.put("messages", prompt);
+ params.put("temperature", 0.95);
+ params.put("top_p", 0.7);
+ headers.put("Authorization", "Bearer "+token);
+ headers.put("Content-Type", "application/json");
+ String html = DownLoadUtil.doPost(url, JSONObject.toJSONString(params),headers);
+ System.out.println(html);
+ Map map = JSONObject.parseObject(html);
+ List> choices = (List>) map.get("choices");
+ Map message = (Map) choices.get(0).get("message");
+ String content = (String) message.get("content");
+ System.out.println(content);
+// List parseArray = JSONObject.parseArray(content, String.class);
+// System.out.println(parseArray);
+// Map result = JSONObject.parseObject(content.replace("\\n", ""));
+// System.out.println(result);
+ }
+}