diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs index 8993ecc..46fad79 100644 --- a/.settings/org.eclipse.core.resources.prefs +++ b/.settings/org.eclipse.core.resources.prefs @@ -1,13 +1,7 @@ -<<<<<<< HEAD +=\=\=\=\=\=\= +<<<<<<<=HEAD +>>>>>>>=branch 'master' of http\://82.156.111.58\:3000/maojian/appliction_manager.git eclipse.preferences.version=1 encoding//src/main/java=UTF-8 encoding//src/main/resources=UTF-8 -encoding//src/test/java=UTF-8 encoding/=UTF-8 -======= -eclipse.preferences.version=1 -encoding//src/main/java=UTF-8 -encoding//src/main/resources=UTF-8 -encoding//src/test/java=UTF-8 -encoding/=UTF-8 ->>>>>>> branch 'master' of http://82.156.111.58:3000/maojian/appliction_manager.git diff --git a/pom.xml b/pom.xml index b93bdce..157fe7a 100644 --- a/pom.xml +++ b/pom.xml @@ -10,10 +10,10 @@ 2.2.4.RELEASE com.bfd - bigmodel + appliction_manager 0.0.1-SNAPSHOT - bigmodel + appliction_manager http://www.example.com @@ -30,142 +30,31 @@ 4.11 test + - org.springframework.boot - spring-boot-starter-web - - - - de.codecentric - spring-boot-admin-starter-client - 2.2.4 - - - com.google.code.gson - gson - 2.8.8 + de.codecentric + spring-boot-admin-starter-server + 2.2.4 + org.springframework.boot - spring-boot-test - - - - org.springframework - spring-test - 5.0.10.RELEASE - test + spring-boot-starter-mail + 2.2.4.RELEASE + - commons-io - commons-io - 1.4 + org.projectlombok + lombok + 1.18.16 + provided com.alibaba fastjson 2.0.17 - - - com.mchange - c3p0 - 0.9.5.5 - - - mysql - mysql-connector-java - 8.0.29 - - - - com.squareup.okhttp3 - okhttp - 4.9.3 - - - org.apache.httpcomponents - httpclient - 4.5.3 - - - commons-lang - commons-lang - 2.6 - - - - org.jetbrains.kotlin - kotlin-reflect - 1.6.21 - runtime - - - - org.jsoup - jsoup - 1.8.1 - - - - org.projectlombok - lombok - - - org.springframework.kafka - spring-kafka - - - - - cn.hutool - hutool-all - 5.8.5 - - - junit - junit - - - - p6spy - p6spy - 3.9.0 - - - commons-collections - commons-collections - 3.2.2 - - - org.elasticsearch.client - transport - 6.0.0 - - - - org.elasticsearch - elasticsearch - 6.0.0 - - - org.elasticsearch.plugin - transport-netty4-client - 6.0.0 - - - - com.auth0 - java-jwt - 4.2.2 - - - com.bfd.util - pauseTool - 1.0 - @@ -217,7 +106,7 @@ org.springframework.boot spring-boot-maven-plugin - com.bfd.qanda.Application + com.bfd.manager.Application ZIP diff --git a/src/main/java/com/bfd/qanda/Application.java b/src/main/java/com/bfd/qanda/Application.java deleted file mode 100644 index 9c1d177..0000000 --- a/src/main/java/com/bfd/qanda/Application.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.bfd.qanda; - - - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.kafka.annotation.EnableKafka; -import org.springframework.scheduling.annotation.EnableScheduling; - -/** - * 主入口 - * - * @author jian.mao - * @date 2023年7月4日 - * @description - */ -@SpringBootApplication -@EnableKafka -public class Application { - - public static void main(String[] args) { - SpringApplication.run(Application.class, args); - } -} \ No newline at end of file diff --git a/src/main/java/com/bfd/qanda/cache/ConfigCache.java b/src/main/java/com/bfd/qanda/cache/ConfigCache.java deleted file mode 100644 index 383daac..0000000 --- a/src/main/java/com/bfd/qanda/cache/ConfigCache.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.bfd.qanda.cache; - -import lombok.extern.slf4j.Slf4j; - -import java.util.Map; -import java.util.concurrent.LinkedBlockingDeque; - -/** - * @author jian.mao - * @date 2022年11月11日 - * @description 静态变量类 - */ -@Slf4j -public class ConfigCache { - - /**启动条件**/ - public static boolean isStart = true; - /*****任务队列*****/ - public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>(); - - - /** - * 队列录入任务 - * @param queue - * @param task - */ - public static void putQueue(LinkedBlockingDeque> queue,Map task){ - //next app 写入队列准备调出 - try { - queue.put(task); - } catch (InterruptedException e) { - log.error("队列写入data失败---"); - } - } -} diff --git a/src/main/java/com/bfd/qanda/cache/LocalCache.java b/src/main/java/com/bfd/qanda/cache/LocalCache.java deleted file mode 100644 index c692107..0000000 --- a/src/main/java/com/bfd/qanda/cache/LocalCache.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.bfd.qanda.cache; - -import java.util.Calendar; -import java.util.Date; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * token 缓存 - * @author jian.mao - * @date 2023年11月8日 - * @description - */ -@Slf4j -public class LocalCache { - - private static final ConcurrentMap CACHE = new ConcurrentHashMap<>(64); - - private LocalCache() { - } - - public static LocalCache getInstance() { - return Inner.LOCAL_CACHE; - } - - public static String get(String key) { - Value v = LocalCache.CACHE.get(key); - if (v == null || new Date().after(v.end)) { - return ""; - } - - log.debug("get key:{},time left:{}s", key, (v.end.getTime() - new Date().getTime()) / 1000); - return v.value; - } - - public static void set(String key, String value, int expire, TimeUnit timeUnit) { - Calendar calendar = Calendar.getInstance(); - calendar.add(Calendar.SECOND, (int) timeUnit.toSeconds(expire)); - Value v = new Value(value, calendar.getTime()); - log.debug("put key:{}, expire time:{} ", key, calendar.getTime()); - LocalCache.CACHE.put(key, v); - } - - private static class Value { - - String value; - Date end; - - public Value(String value, Date time) { - this.value = value; - this.end = time; - } - } - - private static class Inner { - - private static final LocalCache LOCAL_CACHE = new LocalCache(); - } -} diff --git a/src/main/java/com/bfd/qanda/controller/QandaController.java b/src/main/java/com/bfd/qanda/controller/QandaController.java deleted file mode 100644 index a0a44a8..0000000 --- a/src/main/java/com/bfd/qanda/controller/QandaController.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.bfd.qanda.controller; - -import javax.annotation.Resource; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; - -import com.bfd.qanda.service.QandaService; - - -/** - * @author jian.mao - * @date 2023年11月8日 - * @description - */ -@Controller -@RequestMapping("/qanda") -@Slf4j -public class QandaController { - - @Resource - private QandaService qandaService; - @PostMapping("/putQuestion") - @ResponseBody - public String putQuestion(@RequestBody String dataJson){ - String response = qandaService.putQuestion(dataJson); - return response; - } - - @RequestMapping(value = "/hello", method = RequestMethod.GET) - @ResponseBody - public String hello(String param, String token) { - return "123"; - } -} diff --git a/src/main/java/com/bfd/qanda/entity/Constants.java b/src/main/java/com/bfd/qanda/entity/Constants.java deleted file mode 100644 index 2e5b416..0000000 --- a/src/main/java/com/bfd/qanda/entity/Constants.java +++ /dev/null @@ -1,152 +0,0 @@ -package com.bfd.qanda.entity; - - -/** - * 常量实体类 - * @author jian.mao - * @date 2022年11月15日 - * @description - */ -public class Constants { - - /*************************蓝图常量key名称*********************************/ - public final static String SCHEDULING = "scheduling"; - public final static String TYPE = "type"; - public final static String INTERVAL = "interval"; - public final static String CREATED = "created"; - public final static String LAST_EDIT = "last_edit"; - public final static String BLUEPRINT_ID = "blueprint_id"; - public final static String BLUEPRINTID = "blueprintId"; - public final static String BLUEPRINT_NAME = "name"; - public final static String SCENARIO = "scenario"; - public final static String AUTOCOMMITTRIGGERLAST = "autoCommitTriggerLast"; - public final static String FRESHVARIABLES = "freshVariables"; - public final static String AUTOCOMMIT = "autoCommit"; - public final static String MAXERRORS = "maxErrors"; - public final static String DATALOSS = "dataloss"; - public final static String POSITION = "position"; - public final static String SCENES_ID = "scenes_id"; - public final static String SCENESID = "scenesId"; - public final static String MULTI_BRANCH = "multi_branch"; - - public final static String SINGLE = "single"; - /**已重试次数**/ - public final static String ERROR_TIME = "error_time"; - public final static String PREVIOUS_RESULT = "previous_result"; - - /****数据id*****/ - public final static String BUSINESSKEY = "businessKey"; - - - /*************************metadata常量key名称*********************************/ - public final static String LABEL_COL = "label_col"; - public final static String LABEL = "label"; - public final static String USER = "user"; - public final static String ADMIN = "admin"; - public final static String ADDRESS = "address"; - public final static String DATASOURCE = "datasource"; - public final static String INDEX = "index"; - - /*************************app常量key名称*********************************/ - public final static String APPS = "apps"; - public final static String TRANSFER_ID = "transfer_id"; - public final static String MODULE = "module"; - public final static String VERSION = "version"; - public final static String METADATA = "metadata"; - public final static String APP_NAME = "name"; - public final static String DESCRIBE = "describe"; - public final static String NEXT_APP_ID = "next_app_id"; - public final static String EDGE_ID = "edge_id"; - public final static String START_ID = "start_id"; - public final static String END_ID = "end_id"; - - public final static String WAIT_CONDITION = "wait_condition"; - public final static String START_TAG = "start_tag"; - - /*************************module类型*********************************/ - public final static String FILE = "file"; - public final static String OCR = "OCR"; - public final static String FILTER = "Filter"; - public final static String CHATGPT = "ChatGPT"; - public final static String MYSQL = "mysql"; - - /*************************other类型*********************************/ - public final static String UNDERLINE = "_"; - public final static String RESULT_TOPIC = null; - public static final String EMPTY = ""; - public static final String HTTP = "http"; - public static final String REQUEST_ERROR_MESSAGE = "Download failed error is"; - public static final String REQUEST_RESULT = "result"; - public static final String REQUEST_RESULT_RESULTS = "results"; - public static final String MAP_TYPE = "Map"; - public static final String LIST_TYPE = "List"; - public static final String STRING_TYPE = "String"; - public static final String DOCUMENT_TYPE = "doc"; - public static final String FILTER_ZH = "过滤器"; - - public static final String JSON_SELE_SYMBOL = "$."; - public static final String LEFT_BRACKETS = "["; - public static final String RIGTH_BRACKETS = "]"; - public static final String TASKTYPE = "taskType"; - public static final Integer USER_TYPE = 1; - public static final Integer KEYWORD_TYPE = 0; - public static final Integer DETAIL_TYPE = 2; - public static final String CID = "cid"; - public static final String SITETYPE = "siteType"; - public static final Integer DEFULT_SUBJECTID = 304864; - public static final Integer DEFULT_CRAWLCYCLICITYTIME = 1440; - public static final String CRAWLENDTIME = "crawlEndTime"; - public static final String CRAWLSTARTTIME = "crawlStartTime"; - public static final String CRAWLPAGETYPES = "crawlPageTypes"; - public static final String APPID = "113ic"; - public static final String APP_ID = "appId"; - public final static String ID = "id"; - public static final Integer DEFULT_CRAWLPERIODHOUR = 24; - public static final String CREATEUSERID = "662015832180933762"; - public static final String CRAWL_ADD_URL = "https://caiji.percent.cn/api/crawl/remote/task/save"; - public static final String CRAWLKEYWORD = "crawlKeyword"; - public static final String ATTACHTAG = "attachTag"; - public static final String ATTACHTAG_VALUE = "analyze"; - public static final String KEYWORD = "keyword"; - public static final String SITEID = "siteId"; - public static final String RESULTS = "results"; - public static final String RESULT = "result"; - public static final String CRAWLDATAFLAG = "crawlDataFlag"; - public static final String CRAWLDATAFLAG_PREFIX = "\"crawlDataFlag\":\"keyword:"; - public static final String TID = "tid"; - public static final Long TIME_OUT = 1800000L; - public static final String ATTR = "attr"; - public static final String HASVIDEO = "hasVideo"; - public static final String CRAWL_END_MARK = "crawl_end_mark"; - public static final String CRAWL_END_MESSAGE = "crawl_end_message"; - public static final String CRAWL_END_MESSAGE_VALUE = "数据采集完成"; - public static final String SUBJECTID = "subjectId"; - public static final String TASKID = "taskId"; - public static final int SUCCESS_CODE = 200; - public static final String WEB_URL_SUFFIX = "/api/aogeo/api/cda/caiji/status"; - public static final String STATUS = "status"; - /************************redis*************************************/ - public static final String LOCK_KEY = "myLock"; - public static final long LOCK_EXPIRE_TIME = 300000; - - /************************应用参数*************************************/ - public static final String CODE = "code"; - public static final String MESSAGE = "message"; - public static final String INPUT = "input"; - public static final String OUTPUT = "output"; - public static final String FORM = "form"; - public static final String FIELD = "field"; - public static final String VALUE = "value"; - public static final String DATA = "data"; - public static final String COLON_EN = ":"; - /******************************admin*******************************/ - public static final String AUTHORIZATION = "authorization"; - public static final String TEMPERATURE = "temperature"; - public static final String FIELDTYPE = "fieldType"; - public static final String TOP_P = "top_p"; - public static final String MODEL = "model"; - public static final String PROMPT = "prompt"; - public static final String CHATGLM_TURBO_URL = "https://open.bigmodel.cn/api/paas/v4/chat/completions"; - public static final String CHOICES = "choices"; - public static final String CONTENT = "content"; -} diff --git a/src/main/java/com/bfd/qanda/handler/MainHandler.java b/src/main/java/com/bfd/qanda/handler/MainHandler.java deleted file mode 100644 index c4a0365..0000000 --- a/src/main/java/com/bfd/qanda/handler/MainHandler.java +++ /dev/null @@ -1,128 +0,0 @@ -package com.bfd.qanda.handler; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.LinkedBlockingDeque; - -import javax.annotation.Resource; - -import com.bfd.util.PauseTool; -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.io.FileUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.ApplicationArguments; -import org.springframework.boot.ApplicationRunner; -import org.springframework.core.annotation.Order; -import org.springframework.data.redis.core.StringRedisTemplate; -import org.springframework.stereotype.Component; - -import com.alibaba.fastjson.JSONObject; -import com.bfd.qanda.cache.ConfigCache; -import com.bfd.qanda.process.QandaPorcess; -import com.bfd.qanda.utils.FileUtil; - - -/** - * 启动处理入口 - * - * @author jian.mao - * @date 2023年11月3日 - * @description - */ - -@Component -@Order(value = 1) -@Slf4j -public class MainHandler implements ApplicationRunner { - - @Value("${task.task-queue-path}") - private String taskPath; - @Value("${task.thread-num}") - private Integer threadNum; - @Resource - QandaPorcess qandaPorcess; - @Resource - private StringRedisTemplate stringRedisTemplate; - @Value("${zookeeper.connection-string}") - private String connectionString; - @Value("${zookeeper.publish-node}") - private String nodePath; - - @Override - public void run(ApplicationArguments args) throws Exception { - PauseTool pauseTool = new PauseTool(); - pauseTool.initializeRedisCache(stringRedisTemplate); - pauseTool.setupZookeeperListener(connectionString, nodePath); - for (Integer i = 0; i < threadNum; i++) { - //开启问答处理线程 - new Thread(qandaPorcess).start(); - log.info("开启问答处理线程-----"); - } - - //停止处理 - waitDown(); - //启动加载缓存任务 - readTask(taskPath, ConfigCache.taskQueue); - } - - - @SuppressWarnings("unchecked") - public static void readTask(String path, LinkedBlockingDeque> queue) { - File file = new File(path); - if (file.exists()) { - List tasks = null; - try { - tasks = FileUtils.readLines(file, "UTF-8"); - } catch (IOException e) { - e.printStackTrace(); - } - for (String taskStr : tasks) { - Map task = JSONObject.parseObject(taskStr); - try { - queue.put(task); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - file.delete(); - } - } - - /** - * 结束触发钩子 - */ - public void waitDown() { - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - // 停止线程 - ConfigCache.isStart = false; - log.info("stop-------"); - writeTsskToFile(); - } - }); - } - - - /** - * 任务持久化到硬盘 - */ - public void writeTsskToFile() { - while (true) { - if (ConfigCache.taskQueue.size() > 0) { - try { - Map task = ConfigCache.taskQueue.take(); - FileUtil.writeFile(taskPath, JSONObject.toJSONString(task)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } else { - log.info("taskQueue write is file end"); - break; - } - } - } -} diff --git a/src/main/java/com/bfd/qanda/process/QandaPorcess.java b/src/main/java/com/bfd/qanda/process/QandaPorcess.java deleted file mode 100644 index f046dcc..0000000 --- a/src/main/java/com/bfd/qanda/process/QandaPorcess.java +++ /dev/null @@ -1,193 +0,0 @@ -package com.bfd.qanda.process; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import com.bfd.qanda.utils.GPTResultParseUtil; -import com.bfd.util.PauseTool; -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - - -import com.alibaba.fastjson.JSONObject; -import com.bfd.qanda.cache.ConfigCache; -import com.bfd.qanda.entity.Constants; -import com.bfd.qanda.token.TokenManagerV3; -import com.bfd.qanda.utils.DataUtil; -import com.bfd.qanda.utils.DownLoadUtil; -import com.bfd.qanda.utils.SpringBootKafka; - -/** - * @author jian.mao - * @date 2023年11月8日 - * @description - */ -@Component -@Slf4j -public class QandaPorcess implements Runnable { - private static final int CHAT_TYPE_ONE = 1; - @Autowired - private SpringBootKafka springBootKafka; - @Value("${customize-kafka.producer.topic}") - private String topic; - - @Override - public void run() { - while (ConfigCache.isStart) { - Map task = null; - try { - task = ConfigCache.taskQueue.take(); - //结果接收 - Map result = new HashMap(16); - //结果内容 - StringBuffer chatContent = new StringBuffer(); - //输入配置 - Map input = (Map) task.get(Constants.INPUT); - //输出配置 - Map output = (Map) task.get(Constants.OUTPUT); - //数据源 - Map data = (Map) task.get(Constants.DATA); - int scenesId = (int) task.get(Constants.SCENES_ID); - int version = (int) task.get(Constants.VERSION); - String pauseKey = scenesId + "_" + version; - if (!PauseTool.CACHE.containsKey(pauseKey)) { - log.info("流程:{}的版本:{}已失效,任务跳过", scenesId, version); - continue; - } - //请求身份校验 api-key - String authorization = (String) input.get(Constants.AUTHORIZATION); - //fieldType:自定义输出字段: 0 关闭,1-开启,如果开启则拼接form到output里(如果关闭,则取默认的output拼接) - int fieldType = (int) input.get(Constants.FIELDTYPE); - Float temperature = Float.valueOf(input.get(Constants.TEMPERATURE).toString()); - Float topP = Float.valueOf(input.get(Constants.TOP_P).toString()); - String model = (String) input.get(Constants.MODEL).toString(); - List> prompt = (List>) input.get(Constants.PROMPT); - String answerStr = qandaRequest(authorization, model, temperature, topP, prompt, data); - log.info("answerStr:" + answerStr); - Map answer = JSONObject.parseObject(answerStr); - try { - //请求成功,正常解析 - List> choices = (List>) answer.get(Constants.CHOICES); - Map message = (Map) choices.get(0).get(Constants.MESSAGE); - chatContent.append(message.get(Constants.CONTENT)); - } catch (Exception e) { - log.error("问答接口响应体异常:{}", answerStr, e); - // TODO: handle exception - continue; - } - Map results = new HashMap(16); - results.put(Constants.ID, UUID.randomUUID().toString()); - results.put(Constants.CONTENT, chatContent.toString()); - if (fieldType != 0) { - results.remove(Constants.CONTENT); - try { - //请求成功,正常解析 - List> choices = (List>) answer.get(Constants.CHOICES); - Map message = (Map) choices.get(0).get(Constants.MESSAGE); - String reponseContent = (String) message.get(Constants.CONTENT); - Map stringObjectMap = GPTResultParseUtil.parseGPTResult(output, reponseContent); - results.putAll(stringObjectMap); - } catch (Exception e) { - log.error("问答接口响应体异常:{}", answerStr, e); - // TODO: handle exception - continue; - } - } - results.put("isLast", 1); - result.put(Constants.RESULTS, JSONObject.toJSONString(results)); - result.put(Constants.MESSAGE, "成功"); - result.put(Constants.STATUS, 1); - task.put(Constants.RESULT, result); -// //发送kafka - springBootKafka.send(topic, JSONObject.toJSONString(task)); - log.info("数据流转至下游-------"); - } catch (Exception e) { - log.error("问答处理异常,", e); - //结果集 - Map results = new HashMap(16); - - Map result = new HashMap(16); - //遍历入库返回结果,拼接响应内容 - results.put("isLast", 1); - results.put("content", e.getMessage()); - result.put(Constants.RESULTS, JSONObject.toJSONString(results)); - result.put(Constants.MESSAGE, "异常"); - result.put(Constants.STATUS, 2); - task.put(Constants.RESULT, result); - //发送kafka - springBootKafka.send(topic, JSONObject.toJSONString(task)); - log.info("数据流转至下游-------"); - - - } - - } - } - - /** - * 问题请求 - * - * @param authorization - * @param temperature - * @param topP - * @param prompt - * @return - */ - private String qandaRequest(String authorization, String model, Float temperature, Float topP, List> prompt, Map data) { - //新建聊天话术 - StringBuffer chatContent = new StringBuffer(); - for (Map map : prompt) { - if (Integer.valueOf(map.get(Constants.TYPE).toString()) == CHAT_TYPE_ONE) { - chatContent.append(map.get(Constants.VALUE)); - } else { - String jsonPath = (String) map.get(Constants.VALUE); - chatContent.append(DataUtil.getValue(jsonPath, data).toString()); - } - } - //拆分authorization - String[] apiCombination = authorization.split("\\."); - //获取token - String token = TokenManagerV3.getToken(apiCombination[0], apiCombination[1]); - StringBuffer encryptionAuthorization = new StringBuffer("Bearer "); - encryptionAuthorization.append(token); - Map headers = new HashMap(16); - Map params = new HashMap(16); - List> prompts = buildParam(chatContent.toString(), data); - - params.put("model", model); - params.put("messages", prompts); - params.put("temperature", temperature); - params.put("top_p", topP); - - headers.put("Authorization", encryptionAuthorization.toString()); - headers.put("Content-Type", "application/json"); - String html = DownLoadUtil.doPost(Constants.CHATGLM_TURBO_URL, JSONObject.toJSONString(params), headers); - return html; - } - - /** - * 参数构建 - * - * @param prompt - * @param data - * @return - */ - private List> buildParam(String prompt, Map data) { - //聊天体 - List> prompts = new ArrayList>(); - log.info("question---{}", prompt); - Map chat1 = new HashMap(16); - chat1.put("role", "user"); - chat1.put("content", prompt); - prompts.add(chat1); - return prompts; - } - - -} diff --git a/src/main/java/com/bfd/qanda/service/QandaService.java b/src/main/java/com/bfd/qanda/service/QandaService.java deleted file mode 100644 index ce5f79e..0000000 --- a/src/main/java/com/bfd/qanda/service/QandaService.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.bfd.qanda.service; - -/** - * 问答业务层 - * @author jian.mao - * @date 2023年11月8日 - * @description - */ -public interface QandaService { - - /** - * 问答 - * @param dataJson 接口参数 - * @return - */ - public String putQuestion(String dataJson); -} diff --git a/src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java b/src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java deleted file mode 100644 index 98a205b..0000000 --- a/src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.bfd.qanda.service.impl; - -import java.util.HashMap; -import java.util.Map; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Service; - -import com.alibaba.fastjson.JSONObject; -import com.bfd.qanda.cache.ConfigCache; -import com.bfd.qanda.entity.Constants; -import com.bfd.qanda.service.QandaService; - -/** - * 业务层实现类 - * - * @author jian.mao - * @date 2023年11月8日 - * @description - */ -@Service -@Slf4j -public class QandaServiceImpl implements QandaService { - - @Override - public String putQuestion(String dataJson) { - String trace = "trace"; - Map response = new HashMap<>(16); - int code = 200; - String message = "success"; - Map task = null; - try { - task = JSONObject.parseObject(dataJson); - } catch (Exception e) { - log.error("参数结构不合法,", e); - code = 100010; - message = "参数不合法"; - } - // 写入队列 - try { - if (task.containsKey(trace) && (Boolean) task.get(trace)==true) { - log.info("测试流程,插入队首"); - ConfigCache.taskQueue.putFirst(task); - }else { - ConfigCache.taskQueue.put(task); - } - } catch (InterruptedException e) { - log.error("任务写入等待队列异常,", e); - code = 100011; - message = "任务写入等待队列失败"; - } - response.put(Constants.CODE, code); - response.put(Constants.MESSAGE, message); - return JSONObject.toJSONString(response); - } - -} diff --git a/src/main/java/com/bfd/qanda/token/TokenManagerV3.java b/src/main/java/com/bfd/qanda/token/TokenManagerV3.java deleted file mode 100644 index ebe9b89..0000000 --- a/src/main/java/com/bfd/qanda/token/TokenManagerV3.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.bfd.qanda.token; - -import java.util.Calendar; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.algorithms.Algorithm; -import com.bfd.qanda.cache.LocalCache; -import com.bfd.qanda.entity.Constants; - -/** - * token管理类 - * @author jian.mao - * @date 2023年11月8日 - * @description - */ -@Slf4j -public class TokenManagerV3 { - - private static final String TOKENV3KEYPREFIX = "zhipu_oapi_token_v3"; - private static final String ALG = "HS256"; - /***默认过期时间30分钟****/ - private static final int EXPIREMILLIS = 30 * 60 * 1000; - - /** - * token 获取方法 - * @param apiKey 用户id - * @param apiSecret 生成key - * @return - */ - public static String getToken(String apiKey,String apiSecret) { - String tokenCacheKey = genTokenCacheKey(apiKey); - String cacheToken = LocalCache.get(tokenCacheKey); - if(cacheToken != null && !cacheToken.equals(Constants.EMPTY)){ - return cacheToken; - } - String newToken = createJwt(apiKey,apiSecret); - LocalCache.set(tokenCacheKey, newToken, EXPIREMILLIS, TimeUnit.MILLISECONDS); - return newToken; - } - - /** - * token 计算 - * @param apiKey - * @param apiSecret - * @return - */ - private static String createJwt(String apiKey,String apiSecret) { - Algorithm alg; - String hs = "HS256"; - if (ALG.equals(hs)) { - //alg = Algorithm.HMAC256(config.getApiSecret()); - try { - alg = Algorithm.HMAC256(apiSecret.getBytes("utf-8")); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } else { - // 目前仅支持HS256 - log.error("algorithm: %s not supported", ALG); - return null; - } - - Map payload = new HashMap<>(16); - payload.put("api_key", apiKey); - payload.put("exp", System.currentTimeMillis() + EXPIREMILLIS); - payload.put("timestamp", Calendar.getInstance().getTimeInMillis()); - Map headerClaims = new HashMap<>(16); - headerClaims.put("alg", "HS256"); - headerClaims.put("sign_type", "SIGN"); - String token = JWT.create().withPayload(payload).withHeader(headerClaims).sign(alg); - return token; - } - - /** - * 获取缓存key - * @param apiKey - * @return - */ - private static String genTokenCacheKey(String apiKey) { - return String.format("%s-%s", TOKENV3KEYPREFIX, apiKey); - } - public static void main(String[] args) { - String apiKey ="53dbef19a3252a292890408707190f9f"; - String apiSecret="5An47w6PngrTFw19"; - System.out.println(getToken(apiKey,apiSecret)); - } -} diff --git a/src/main/java/com/bfd/qanda/utils/DataUtil.java b/src/main/java/com/bfd/qanda/utils/DataUtil.java deleted file mode 100644 index 6ca3666..0000000 --- a/src/main/java/com/bfd/qanda/utils/DataUtil.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.bfd.qanda.utils; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.JSONPath; - -import java.util.Map; - -/** - * @author:jinming - * @className:DataUtil - * @version:1.0 - * @description: 获取dataValue的值 - * @Date:2023/11/1 9:54 - */ -public class DataUtil { - /** - * - * @param key 传入的key - * @param dataMap 数据map - * @return 根据传入的参数进行判断解析,返回正确的dataValue - */ - public static Object getValue(String key, Map dataMap) { - Object dataValue; - String isJson = "#json#"; - if (key.contains(isJson)) { - //进行第一次拆分,获取#json#前面的部分 - String[] keySplit = key.split(isJson); - String firstDataKey = keySplit[0]; - String[] firstDataKeySplit = firstDataKey.split(":"); - //取出前半部分对应的JSON数据并转换为JSONObject - String dataJson = (String) dataMap.get(firstDataKeySplit[0]); - JSONObject dataJsonObject = JSON.parseObject(dataJson); - //根据key的后半部分取出对应JSONObject中的值 - String firstDataKeyJson = (String) JSONPath.eval(dataJsonObject, firstDataKeySplit[1]); - String secDataKey = keySplit[1]; - JSONObject firstDataJsonObject = JSON.parseObject(firstDataKeyJson); - dataValue = JSONPath.eval(firstDataJsonObject, secDataKey); - return dataValue; - } - String[] keySplit = key.split(":"); - String jsonPath = keySplit[1]; - String dataJson = (String) dataMap.get(keySplit[0]); - JSONObject dataJsonObject = JSON.parseObject(dataJson); - dataValue = JSONPath.eval(dataJsonObject, jsonPath); - return dataValue; - } -} \ No newline at end of file diff --git a/src/main/java/com/bfd/qanda/utils/DateUtil.java b/src/main/java/com/bfd/qanda/utils/DateUtil.java deleted file mode 100644 index 0b42b01..0000000 --- a/src/main/java/com/bfd/qanda/utils/DateUtil.java +++ /dev/null @@ -1,177 +0,0 @@ -package com.bfd.qanda.utils; - - -import java.math.BigInteger; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Date; - -import lombok.extern.slf4j.Slf4j; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; - -/** - * 日期工具类 - * - * @author jian.mao - * @date 2022年11月15日 - * @description - */ -@Slf4j -public class DateUtil { - - /** - * @return - */ - public static String getTimeStrForNow() { - SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH"); - return sdf.format(new Date()); - } - - - public static String getTimeStrForDay(long time) { - SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); - - return sdf.format(new Date(time * 1000)); - } - - public static String getTimeStrForDay() { - SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); - - return sdf.format(new Date()); - } - - - public static String getDateTime() { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - String time = sdf.format(new Date()); - return time; - } - - public static String getDateTime(Long timestap) { - - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - String time = sdf.format(new Date(timestap)); - return time; - } - - public static String getDate(Long timestap) { - - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - String time = sdf.format(new Date(timestap)); - return time; - } - - public static String getDateTimeForMonth() { - SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM"); - String time = sdf.format(new Date()); - return time; - } - - /** - * 休眠 - * - * @param millis 毫秒 - */ - public static void sleep(long millis) { - try { - Thread.sleep(millis); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - /** - * 1. @Description:时间戳转时间 - * 2. @Author: ying.zhao - * 3. @Date: 2023/3/28 - */ - - public static String timestampToDate(String time) { - int thirteen = 13; - int ten = 10; - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); -// if (time.length() == thirteen) { - if (time.length() > ten) { - return sdf.format(new Date(Long.parseLong(time))); - } else { - return sdf.format(new Date(Integer.parseInt(time) * 1000L)); - } - } - - public static String parseCreated(String jsonTime){ - String formattedDateTime = getDateTime(); - try { - // 使用fastjson解析JSON数据 - JSONObject jsonObject = JSON.parseObject(jsonTime); - // 获取日期和时间的值 - JSONObject dateObject = jsonObject.getJSONObject("date"); - int day = dateObject.getIntValue("day"); - int month = dateObject.getIntValue("month"); - int year = dateObject.getIntValue("year"); - - JSONObject timeObject = jsonObject.getJSONObject("time"); - int hour = timeObject.getIntValue("hour"); - int minute = timeObject.getIntValue("minute"); - int second = timeObject.getIntValue("second"); - - // 创建LocalDateTime对象 - LocalDateTime dateTime = LocalDateTime.of(year, month, day, hour, minute, second); - - // 定义日期时间格式化器 - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - - // 格式化日期时间 - formattedDateTime = dateTime.format(formatter); - } catch (Exception e) { - log.info("日期转换失败:{}",e); - } - return formattedDateTime; - } - - /** - * 字符串转换日期 - * @param format - * @param date - * @return - */ - public static Date strToDate(String format,String date){ - SimpleDateFormat sdf = new SimpleDateFormat(format); - if (date == null || date.equals("")){ - return new Date(); - }else{ - Date ru = null; - try { - ru = sdf.parse(date); - } catch (ParseException e) { - e.printStackTrace(); - } - return ru; - } - } - /** - * 日期格式话 - * @param format 日期格式 - * @param dater 要转换的日期,默认当前时间 - * @return - */ - public static String FormatDate(String format,Date date){ - String fromatDate = null; - SimpleDateFormat sdf = new SimpleDateFormat(format); - if (date == null){ - fromatDate = sdf.format(new Date()); - }else{ - fromatDate = sdf.format(date); - } - return fromatDate; - } - public static void main(String[] args) { - String time = timestampToDate("955814400000"); - System.out.println(time); - } -} diff --git a/src/main/java/com/bfd/qanda/utils/DownLoadUtil.java b/src/main/java/com/bfd/qanda/utils/DownLoadUtil.java deleted file mode 100644 index 9f164b2..0000000 --- a/src/main/java/com/bfd/qanda/utils/DownLoadUtil.java +++ /dev/null @@ -1,914 +0,0 @@ -package com.bfd.qanda.utils; - -import java.io.IOException; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; - -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.StatusLine; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.AuthCache; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.HttpClient; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.config.SocketConfig; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.socket.LayeredConnectionSocketFactory; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.auth.BasicScheme; -import org.apache.http.impl.client.BasicAuthCache; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.client.LaxRedirectStrategy; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.bfd.qanda.entity.Constants; - - - - - - - -/** - * 下载工具类 - * @author jian.mao - * @date 2023年9月19日 - * @description - */ -public class DownLoadUtil { - - private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"; - private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class); - /** 代理服务器(产品官网 www.16yun.cn) **/ - final static String PROXYHOST = "u270.40.tp.16yun.cn"; - final static Integer PROXYPORT = 6448; - /** 代理验证信息 **/ - final static String PROXYUSER = "16HFBVJC"; - final static String PROXYPASS = "897944"; - - private static PoolingHttpClientConnectionManager cm = null; - private static HttpRequestRetryHandler httpRequestRetryHandler = null; - private static HttpHost proxy = null; - - private static CredentialsProvider credsProvider = null; - private static RequestConfig reqConfig = null; - - static { - ConnectionSocketFactory plainsf = PlainConnectionSocketFactory - .getSocketFactory(); - LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory - .getSocketFactory(); - - Registry registry = RegistryBuilder.create().register("http", plainsf) - .register("https", sslsf).build(); - - cm = new PoolingHttpClientConnectionManager(registry); - cm.setMaxTotal(20); - cm.setDefaultMaxPerRoute(5); - - proxy = new HttpHost(PROXYHOST, PROXYPORT, "https"); - - credsProvider = new BasicCredentialsProvider(); - credsProvider.setCredentials(AuthScope.ANY, - new UsernamePasswordCredentials(PROXYUSER, PROXYPASS)); - - reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000) - .setConnectTimeout(5000).setSocketTimeout(5000) - .setExpectContinueEnabled(false) - .setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build(); - } - - /** - * 模拟客户端get请求 - * - * @param url - * 模拟请求得url - * @param headers - * 头部信息,没有可以不传 - * @return - */ - @SafeVarargs - public static String proxyDoGet(String url, Map... headers) { - // 设置超时时间 - int timeout = 30; - RequestConfig config = RequestConfig.custom() - .setConnectTimeout(timeout * 1000) - .setConnectionRequestTimeout(timeout * 1000) - .setSocketTimeout(timeout * 1000).build(); - SocketConfig socketConfig = SocketConfig.custom() - .setSoKeepAlive(false) - .setSoLinger(1) - .setSoReuseAddress(true) - .setSoTimeout(timeout * 1000) - .setTcpNoDelay(true).build(); - AuthCache authCache = new BasicAuthCache(); - authCache.put(proxy, new BasicScheme()); - HttpClientContext localContext = HttpClientContext.create(); - localContext.setAuthCache(authCache); - HttpClientBuilder httpBuilder = HttpClientBuilder.create(); - CloseableHttpClient httpClient = httpBuilder - .setDefaultSocketConfig(socketConfig) - .setDefaultRequestConfig(config) - .setDefaultCredentialsProvider(credsProvider).build(); - HttpGet httpGet = new HttpGet(url); - httpGet.setConfig(reqConfig); - if (headers != null && headers.length > 0) { - Map tempHeaders = headers[0]; - for (String key : tempHeaders.keySet()) { - httpGet.setHeader(key, tempHeaders.get(key).toString()); - } - } else { - httpGet.setHeader("Accept", - "application/json, text/javascript, */*; q=0.01"); - httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); - } - CloseableHttpResponse response = null; - String html = ""; - int notFundCode = 404; - int successCode = 200; - try { - response = httpClient.execute(httpGet, localContext); - // 从响应模型中获取响应实体 - HttpEntity responseEntity = response.getEntity(); - StatusLine statusLine = response.getStatusLine(); - System.out.println("响应状态为:" + response.getStatusLine()); - if (statusLine.getStatusCode() == successCode) { - if (responseEntity != null) { - html = EntityUtils.toString(responseEntity, "utf-8"); - System.out.println("响应内容长度为:" - + responseEntity.getContentLength()); - // 下载结果为空不正常 - if (html.equals(Constants.EMPTY)) { - html = "Download failed error is:reslut is null"; - } - } - } else if (statusLine.getStatusCode() == notFundCode) { - html = "

页面404,正常结束请求即可

"; - } else { - throw new Exception("请求错误,code码为:" + statusLine.getStatusCode()); - } - } catch (Exception e) { - e.printStackTrace(); - html = "Download failed error is:reslut is null"; - }finally{ - try { - response.close(); - httpClient.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - return html; - - } - - - public static String httpsslProxyGet(String url, Map... headers) throws Exception { - //采用绕过验证的方式处理https请求 - SSLContext sslcontext = createIgnoreVerifySSL(); - - // 设置协议http和https对应的处理socket链接工厂的对象 - Registry socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.INSTANCE) - .register("https", new SSLConnectionSocketFactory(sslcontext)) - .build(); - PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - connManager.setMaxTotal(50); - connManager.setDefaultMaxPerRoute(10); - HttpClients.custom().setConnectionManager(connManager); - // 设置超时时间 - int timeout = 30; - RequestConfig config = RequestConfig.custom() - .setConnectTimeout(timeout * 1000) - .setConnectionRequestTimeout(timeout * 1000) - .setSocketTimeout(timeout * 1000).build(); - SocketConfig socketConfig = SocketConfig.custom() - .setSoKeepAlive(false) - .setSoLinger(1) - .setSoReuseAddress(true) - .setSoTimeout(timeout * 1000) - .setTcpNoDelay(true).build(); - AuthCache authCache = new BasicAuthCache(); - authCache.put(proxy, new BasicScheme()); - HttpClientContext localContext = HttpClientContext.create(); - localContext.setAuthCache(authCache); - HttpClientBuilder httpBuilder = HttpClientBuilder.create(); - CloseableHttpClient httpClient = httpBuilder - .setConnectionManager(connManager) - .setDefaultSocketConfig(socketConfig) - .setDefaultRequestConfig(config) - .setDefaultCredentialsProvider(credsProvider).build(); - HttpGet httpGet = new HttpGet(url); - httpGet.setConfig(reqConfig); - if (headers != null && headers.length > 0) { - Map tempHeaders = headers[0]; - for (String key : tempHeaders.keySet()) { - httpGet.setHeader(key, tempHeaders.get(key).toString()); - } - } else { - httpGet.setHeader("Accept", - "application/json, text/javascript, */*; q=0.01"); - httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); - } - CloseableHttpResponse response = null; - String html = ""; - int notFundCode = 404; - int successCode = 200; - try { - response = httpClient.execute(httpGet, localContext); - // 从响应模型中获取响应实体 - HttpEntity responseEntity = response.getEntity(); - StatusLine statusLine = response.getStatusLine(); - System.out.println("响应状态为:" + response.getStatusLine()); - if (statusLine.getStatusCode() == successCode) { - if (responseEntity != null) { - html = EntityUtils.toString(responseEntity, "utf-8"); - System.out.println("响应内容长度为:" - + responseEntity.getContentLength()); - // 下载结果为空不正常 - if (html.equals(Constants.EMPTY)) { - html = "Download failed error is:reslut is null"; - } - } - } else if (statusLine.getStatusCode() == notFundCode) { - html = "

页面404,正常结束请求即可

"; - } else { - throw new Exception("请求错误,code码为:" + statusLine.getStatusCode()); - } - } catch (Exception e) { - e.printStackTrace(); - html = "Download failed error is:reslut is null"; - }finally{ - try { - response.close(); - httpClient.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - return html; - - } - - - /** - * json参数方式POST提交 - * @param url - * @param params - * @return - */ - public static String doPost(String url, String params, Map... headers){ - String strResult = ""; - //设置超时时间 - int timeout = 60; - RequestConfig config = RequestConfig.custom(). - setConnectTimeout(timeout * 1000). - setConnectionRequestTimeout(timeout * 1000). - setSocketTimeout(timeout * 1000).build(); - SocketConfig socketConfig = SocketConfig.custom() - .setSoKeepAlive(false) - .setSoLinger(1) - .setSoReuseAddress(true) - .setSoTimeout(timeout * 1000) - .setTcpNoDelay(true).build(); -// AuthCache authCache = new BasicAuthCache(); -// authCache.put(proxy, new BasicScheme()); -// HttpClientContext localContext = HttpClientContext.create(); -// localContext.setAuthCache(authCache); - // 1. 获取默认的client实例 - HttpClientBuilder httpBuilder = HttpClientBuilder.create(); - httpBuilder.setUserAgent(ua); - HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); -// HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) -// .setDefaultCredentialsProvider(credsProvider).build(); - // 2. 创建httppost实例 - HttpPost httpPost = new HttpPost(url); -// httpPost.setConfig(reqConfig); - if (headers != null && headers.length > 0) { - Map tempHeaders = headers[0]; - for (String key : tempHeaders.keySet()) { - httpPost.setHeader(key, tempHeaders.get(key).toString()); - } - } else { - httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); - } - HttpResponse resp = null; - try { - httpPost.setEntity(new StringEntity(params,"utf-8")); - resp = client.execute(httpPost); -// resp = client.execute(httpPost,localContext); - StatusLine statusLine = resp.getStatusLine(); - System.out.println("响应状态为:" + resp.getStatusLine()); - int notFundCode = 404; - int successCode = 200; - if(statusLine.getStatusCode() == successCode){ - // 7. 获取响应entity - HttpEntity respEntity = resp.getEntity(); - strResult = EntityUtils.toString(respEntity, "UTF-8"); - if(strResult.equals(Constants.EMPTY)){ - strResult = "Download failed error is:reslut is null"; - } - }else{ - throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); - } - } catch (Exception e) { - e.printStackTrace(); - strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); - } - return strResult; - } - public static String httpPost(String url,String params) { - String html=""; - html = doPost(url,params); - int i = 1; - while(true){ - if(html.contains("Download failed error is:")){ - log.error("DownLoadUtil------------->download is failure,url is:"+url); - DateUtil.sleep(5000); - i++; - }else{ - break; - } - if(i > 5){ - break; - } - html = doPost(url,params); - } - return html; - } - /** - * 绕过验证 - * - * @return - * @throws NoSuchAlgorithmException - * @throws KeyManagementException - */ - public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException { - SSLContext sc = SSLContext.getInstance("SSLv3"); - - // 实现一个X509TrustManager接口,用于绕过验证,不用修改里面的方法 - X509TrustManager trustManager = new X509TrustManager() { - @Override - public void checkClientTrusted( - java.security.cert.X509Certificate[] paramArrayOfX509Certificate, - String paramString) throws CertificateException { - } - - @Override - public void checkServerTrusted( - java.security.cert.X509Certificate[] paramArrayOfX509Certificate, - String paramString) throws CertificateException { - } - - @Override - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - }; - - sc.init(null, new TrustManager[] { trustManager }, null); - return sc; - } - /** - * 模拟请求 - * - * @param url 资源地址 - * @param map 参数列表 - * @param encoding 编码 - * @return - * @throws NoSuchAlgorithmException - * @throws KeyManagementException - * @throws IOException - * @throws ClientProtocolException - */ - public static String httpsslGet(String url,Map ... headers) { - String html=""; - CloseableHttpClient client = null; - HttpEntity responseEntity = null; - CloseableHttpResponse response = null; - try { - log.debug("DownLoadUtil------------->设置下载相关信息, start...."); - //采用绕过验证的方式处理https请求 - SSLContext sslcontext = createIgnoreVerifySSL(); - - // 设置协议http和https对应的处理socket链接工厂的对象 - Registry socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.INSTANCE) - .register("https", new SSLConnectionSocketFactory(sslcontext)) - .build(); - PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - connManager.setMaxTotal(50); - connManager.setDefaultMaxPerRoute(10); - HttpClients.custom().setConnectionManager(connManager); - //设置超时时间 - int timeout = 30; - RequestConfig config = RequestConfig.custom(). - setConnectTimeout(timeout * 1000). - setConnectionRequestTimeout(timeout * 1000). - setSocketTimeout(timeout * 1000).build(); - SocketConfig socketConfig = SocketConfig.custom() - .setSoKeepAlive(false) - .setSoLinger(1) - .setSoReuseAddress(true) - .setSoTimeout(10000) - .setTcpNoDelay(true).build(); - // 设置重定向策略 - LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy(); - //创建自定义的httpclient对象 - client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build(); -// CloseableHttpClient client = HttpClients.createDefault(); - - HttpGet httpGet = new HttpGet(url); - if(headers != null && headers.length > 0){ - Map tempHeaders = headers[0]; - for (String key : tempHeaders.keySet()) { - httpGet.setHeader(key,tempHeaders.get(key).toString()); - } - }else{ - httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); - httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); - httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); - } - log.debug("DownLoadUtil------------->设置下载相关信息, end...."); - try { - int notFundCode = 404; - int successCode = 200; - log.debug("DownLoadUtil------------->下载执行,start...."); - httpGet.setConfig(config); - response = client.execute(httpGet); - log.debug("DownLoadUtil------------->下载执行,end...."); - // 从响应模型中获取响应实体 - StatusLine statusLine = response.getStatusLine(); - log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+",read is start ...."); - System.out.println("响应状态为:" + response.getStatusLine()); - responseEntity = response.getEntity(); - log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+",read is end ...."); - if(statusLine.getStatusCode() == successCode){ - if (responseEntity != null) { - html=EntityUtils.toString(responseEntity,"utf-8"); - System.out.println("响应内容长度为:" + responseEntity.getContentLength()); - } - }else if(statusLine.getStatusCode() == notFundCode){ - html = "

页面404,正常结束请求即可

"; - }else{ - throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); - } - } catch (Exception e) { - e.printStackTrace(); - html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); - - } - } catch (Exception e) { - e.printStackTrace(); - html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); - }finally{ - try { - responseEntity.getContent().close(); - response.close(); - client.close(); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - - return html; - } - - public static String httpSSLGet(String url,Map ... headers) { - String html=""; - html = httpsslGet(url,headers); - int i = 1; - while(true){ - if(html.contains("Download failed error is:")){ - log.error("DownLoadUtil------------->download is failure,url is:"+url); - DateUtil.sleep(30000); - i++; - }else{ - break; - } - if(i > 5){ - break; - } - html = httpsslGet(url,headers); - } - return html; - } - public static String doPostFrom(String url,Map param,Map ... headers){ - //设置超时时间 - int timeout = 15; - RequestConfig config = RequestConfig.custom(). - setConnectTimeout(timeout * 1000). - setConnectionRequestTimeout(timeout * 1000). - setSocketTimeout(timeout * 1000).build(); - SocketConfig socketConfig = SocketConfig.custom() - .setSoKeepAlive(false) - .setSoLinger(1) - .setSoReuseAddress(true) - .setSoTimeout(10000) - .setTcpNoDelay(true).build(); -// AuthCache authCache = new BasicAuthCache(); -// authCache.put(proxy, new BasicScheme()); -// HttpClientContext localContext = HttpClientContext.create(); -// localContext.setAuthCache(authCache); - HttpClientBuilder httpBuilder = HttpClientBuilder.create(); - httpBuilder.setUserAgent(ua); -// HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) -// .setDefaultCredentialsProvider(credsProvider).build(); - HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); - HttpPost httpPost = new HttpPost(url); -// httpPost.setConfig(reqConfig); - if(headers != null && headers.length > 0){ - Map tempHeaders = headers[0]; - for (String key : tempHeaders.keySet()) { - httpPost.setHeader(key,tempHeaders.get(key).toString()); - } - }else{ - httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"); - httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); - httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); - httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"); -// httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html"); - } - // 创建请求参数 - List list = new LinkedList<>(); - for (String key : param.keySet()) { - BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); - list.add(param1); - } - // 使用URL实体转换工具 - String html=""; - try { - UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); - httpPost.setEntity(entityParam); - HttpResponse response = httpClient.execute(httpPost); -// HttpResponse response = httpClient.execute(httpPost,localContext); - // 从响应模型中获取响应实体 - HttpEntity responseEntity = response.getEntity(); - StatusLine statusLine = response.getStatusLine(); - System.out.println("响应状态为:" + response.getStatusLine()); - int notFundCode = 404; - int successCode = 200; - if(statusLine.getStatusCode() == successCode){ - if (responseEntity != null) { - html=EntityUtils.toString(responseEntity,"utf-8"); - } - }else{ - throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); - } - - } catch (Exception e) { - e.printStackTrace(); - html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); - } - - return html; - - } - public static String httpPostForm(String url,Map params,Map ... headers) { - String html=""; - html = doPostFrom(url,params); - int i = 1; - while(true){ - if(html.contains("Download failed error is:")){ - log.error("DownLoadUtil------------->download is failure,url is:"+url); - DateUtil.sleep(5000); - i++; - }else{ - break; - } - if(i > 5){ - break; - } - html = doPostFrom(url,params,headers); - } - return html; - } - - public static String dosslPost(String url,String params,Map ... headers) { - String html=""; - CloseableHttpClient client = null; - HttpEntity responseEntity = null; - CloseableHttpResponse response = null; - try { - //采用绕过验证的方式处理https请求 - SSLContext sslcontext = createIgnoreVerifySSL(); - // 设置协议http和https对应的处理socket链接工厂的对象 - Registry socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.INSTANCE) - .register("https", new SSLConnectionSocketFactory(sslcontext)) - .build(); - PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - HttpClients.custom().setConnectionManager(connManager); - //设置超时时间 - int timeout = 5; - RequestConfig config = RequestConfig.custom(). - setConnectTimeout(timeout * 1000). - setConnectionRequestTimeout(timeout * 1000). - setSocketTimeout(timeout * 1000).build(); - SocketConfig socketConfig = SocketConfig.custom() - .setSoKeepAlive(false) - .setSoLinger(1) - .setSoReuseAddress(true) - .setSoTimeout(10000) - .setTcpNoDelay(true).build(); - //创建自定义的httpclient对象 - client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); -// CloseableHttpClient client = HttpClients.createDefault(); - // 2. 创建httppost实例 - HttpPost httpPost = new HttpPost(url); -// httpPost.setConfig(reqConfig); - httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); - if(headers != null && headers.length > 0){ - Map tempHeaders = headers[0]; - for (String key : tempHeaders.keySet()) { - httpPost.setHeader(key,tempHeaders.get(key).toString()); - } - }else{ - httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); - httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); - httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); - } - - try { - httpPost.setEntity(new StringEntity(params,"utf-8")); - response = client.execute(httpPost); - int notFundCode = 404; - int successCode = 200; - // 从响应模型中获取响应实体 - StatusLine statusLine = response.getStatusLine(); - System.out.println("响应状态为:" + response.getStatusLine()); - responseEntity = response.getEntity(); - if(statusLine.getStatusCode() == successCode){ - if (responseEntity != null) { - html=EntityUtils.toString(responseEntity,"utf-8"); - System.out.println("响应内容长度为:" + responseEntity.getContentLength()); - } - }else if(statusLine.getStatusCode() == notFundCode){ - html = "

页面404,正常结束请求即可

"; - }else{ - throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); - } - } catch (Exception e) { - e.printStackTrace(); - html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); - - } - } catch (Exception e) { - e.printStackTrace(); - html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); - }finally{ - try { - responseEntity.getContent().close(); - response.close(); - client.close(); - } catch (UnsupportedOperationException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - - } - - - return html; - } - public static String dosslPostForm(String url,Map param,Map ... headers) { - String html=""; - try { - //采用绕过验证的方式处理https请求 - SSLContext sslcontext = createIgnoreVerifySSL(); - - // 设置协议http和https对应的处理socket链接工厂的对象 - Registry socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.INSTANCE) - .register("https", new SSLConnectionSocketFactory(sslcontext)) - .build(); - PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - HttpClients.custom().setConnectionManager(connManager); - //设置超时时间 - int timeout = 5; - RequestConfig config = RequestConfig.custom(). - setConnectTimeout(timeout * 1000). - setConnectionRequestTimeout(timeout * 1000). - setSocketTimeout(timeout * 1000).build(); - SocketConfig socketConfig = SocketConfig.custom() - .setSoKeepAlive(false) - .setSoLinger(1) - .setSoReuseAddress(true) - .setSoTimeout(10000) - .setTcpNoDelay(true).build(); - //创建自定义的httpclient对象 - CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); -// CloseableHttpClient client = HttpClients.createDefault(); - // 2. 创建httppost实例 - HttpPost httpPost = new HttpPost(url); -// httpPost.setConfig(reqConfig); - if(headers != null && headers.length > 0){ - Map tempHeaders = headers[0]; - for (String key : tempHeaders.keySet()) { - httpPost.setHeader(key,tempHeaders.get(key).toString()); - } - }else{ - httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); - httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); - httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); - httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); - } - - // 创建请求参数 - List list = new LinkedList<>(); - for (String key : param.keySet()) { - BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); - list.add(param1); - } - // 使用URL实体转换工具 - try { - UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); - httpPost.setEntity(entityParam); - HttpResponse response = client.execute(httpPost); -// HttpResponse response = httpClient.execute(httpPost,localContext); - // 从响应模型中获取响应实体 - int notFundCode = 404; - int successCode = 200; - HttpEntity responseEntity = response.getEntity(); - StatusLine statusLine = response.getStatusLine(); - System.out.println("响应状态为:" + response.getStatusLine()); - if(statusLine.getStatusCode() == successCode){ - if (responseEntity != null) { - html=EntityUtils.toString(responseEntity,"utf-8"); - } - }else{ - throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); - } - - } catch (Exception e) { - e.printStackTrace(); - html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); - } - } catch (Exception e) { - e.printStackTrace(); - html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); - } - - - return html; - } - public static String httpSSLPostForm(String url,Map params,Map ...headers) { - String html=""; - try { - html = dosslPostForm(url,params,headers); - } catch (Exception e) { - e.printStackTrace(); - // TODO: handle exception - html = "Download failed error is:Exception!"; - } - int i = 1; - while(true){ - if(html.contains("Download failed error is:")){ - log.error("DownLoadUtil------------->download is failure,url is:"+url); - DateUtil.sleep(30000); - i++; - }else{ - break; - } - if(i > 5){ - break; - } - try { - html = dosslPostForm(url,params,headers); - } catch (Exception e) { - e.printStackTrace(); - // TODO: handle exception - html = "Download failed error is:Exception!"; - } - } - return html; - } - public static String httpSSLPost(String url,String params,Map ...headers) { - String html=""; - try { - html = dosslPost(url,params,headers); - } catch (Throwable e) { - e.printStackTrace(); - // TODO: handle exception - html = "Download failed error is:Exception!"; - } - int i = 1; - while(true){ - if(html.contains("Download failed error is:")){ - log.error("DownLoadUtil------------->download is failure,url is:"+url); - DateUtil.sleep(30000); - i++; - }else{ - break; - } - if(i > 5){ - break; - } - try { - html = dosslPost(url,params,headers); - } catch (Throwable e) { - e.printStackTrace(); - // TODO: handle exception - html = "Download failed error is:Exception!"; - } - } - return html; - } - - /** - * 模拟客户端get请求 - * @param url 模拟请求得url - * @param headers 头部信息,没有可以不传 - * @return - */ - public static String doGet(String url,Map ... headers){ - //设置超时时间 - int timeout = 15; - RequestConfig config = RequestConfig.custom(). - setConnectTimeout(timeout * 1000). - setConnectionRequestTimeout(timeout * 1000). - setSocketTimeout(timeout * 1000).build(); - SocketConfig socketConfig = SocketConfig.custom() - .setSoKeepAlive(false) - .setSoLinger(1) - .setSoReuseAddress(true) - .setSoTimeout(10000) - .setTcpNoDelay(true).build(); - HttpClientBuilder httpBuilder = HttpClientBuilder.create(); - httpBuilder.setUserAgent(ua); - HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); - HttpGet httpGet = new HttpGet(url); - if(headers != null && headers.length > 0){ - Map tempHeaders = headers[0]; - for (String key : tempHeaders.keySet()) { - httpGet.setHeader(key,tempHeaders.get(key).toString()); - } - }else{ - httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); - httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); - } - String html=""; - try { - int notFundCode = 404; - int successCode = 200; - HttpResponse response = httpClient.execute(httpGet); - // 从响应模型中获取响应实体 - HttpEntity responseEntity = response.getEntity(); - StatusLine statusLine = response.getStatusLine(); - System.out.println("响应状态为:" + response.getStatusLine()); - if(statusLine.getStatusCode() == successCode){ - if (responseEntity != null) { - html=EntityUtils.toString(responseEntity,"utf-8"); - if(html.equals("")){ - html = "Download failed error is:reslut is null"; - } - } - }else if(statusLine.getStatusCode() == notFundCode){ - html = "

页面404,正常结束请求即可

"; - }else{ - throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); - } - } catch (Exception e) { - e.printStackTrace(); - html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); - } - return html; - - } - public static void main(String[] args) throws Exception { - - } -} diff --git a/src/main/java/com/bfd/qanda/utils/EncryptionUtil.java b/src/main/java/com/bfd/qanda/utils/EncryptionUtil.java deleted file mode 100644 index 1efdef6..0000000 --- a/src/main/java/com/bfd/qanda/utils/EncryptionUtil.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.bfd.qanda.utils; - -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -/** - * @author jian.mao - * @date 2023年3月10日 - * @description - */ -public class EncryptionUtil { - public static String md5(String text) { - try { - MessageDigest md = MessageDigest.getInstance("MD5"); - md.update(text.getBytes()); - byte[] bytes = md.digest(); - StringBuilder sb = new StringBuilder(); - for (byte b : bytes) { - sb.append(String.format("%02x", b & 0xff)); - } - return sb.toString(); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - return null; - } - } -} diff --git a/src/main/java/com/bfd/qanda/utils/FileUtil.java b/src/main/java/com/bfd/qanda/utils/FileUtil.java deleted file mode 100644 index 7e40c19..0000000 --- a/src/main/java/com/bfd/qanda/utils/FileUtil.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.bfd.qanda.utils; - -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * 文件工具类 - * @author jian.mao - * @date 2023年7月14日 - * @description - */ -public class FileUtil { - - /** - * 数据写入文件 - * @param Path 文件路径 - * @param result 数据 - * @throws IOException - */ - public static void writeFile(String path,String result){ - try { - FileWriter fw = new FileWriter(path,true); - fw.write(result+"\n"); - fw.flush(); - fw.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } -} diff --git a/src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java b/src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java deleted file mode 100644 index 3bd86df..0000000 --- a/src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.bfd.qanda.utils; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.JSONException; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author:jinming - * @className:GPTResultParseUtil - * @version:1.0 - * @description: - * @Date:2024/6/28 10:11 - */ -public class GPTResultParseUtil { - public static Map parseGPTResult(Map output, String gptContent) { - Map jsonResult = new HashMap<>(); - try { - // 替换```json, ``` 和 \n - String jsonContent = gptContent.replace("```json", "").replace("```", "").replace("\n", ""); - JSONObject jsonGPT = JSON.parseObject(jsonContent); - - for (String key : output.keySet()) { - if (jsonGPT.containsKey(key)) { - jsonResult.put(key, jsonGPT.get(key)); - } - } - return jsonResult; - } catch (JSONException e) { - try { - // 直接解析失败,使用正则表达式匹配外层的 {} - Pattern pattern = Pattern.compile("\\{.*\\}", Pattern.DOTALL); - Matcher matcher = pattern.matcher(gptContent.replace("\n", "")); - if (matcher.find()) { - JSONObject jsonGPT = JSON.parseObject(matcher.group()); - for (String key : output.keySet()) { - if (jsonGPT.containsKey(key)) { - jsonResult.put(key, jsonGPT.get(key)); - } - } - return jsonResult; - } else { - return null; - } - } catch (Exception ex) { - ex.printStackTrace(); - return null; - } - } - } -} \ No newline at end of file diff --git a/src/main/java/com/bfd/qanda/utils/JsonUtil.java b/src/main/java/com/bfd/qanda/utils/JsonUtil.java deleted file mode 100644 index 46fb9fb..0000000 --- a/src/main/java/com/bfd/qanda/utils/JsonUtil.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.bfd.qanda.utils; - -import com.alibaba.fastjson.JSONObject; -import com.bfd.qanda.entity.Constants; - -/** - * json工具 - * @author jian.mao - * @date 2023年7月10日 - * @description - */ -public class JsonUtil { - - /** - * 校验字符串是list/map/str - * @param jsonString - * @return - */ - public static String checkJsonType(String jsonString) { - try { - JSONObject.parseObject(jsonString); - return Constants.MAP_TYPE; - } catch (Exception e) { - try { - JSONObject.parseArray(jsonString); - return Constants.LIST_TYPE; - } catch (Exception ex) { - return Constants.STRING_TYPE; - } - } - } -} diff --git a/src/main/java/com/bfd/qanda/utils/OtherUtils.java b/src/main/java/com/bfd/qanda/utils/OtherUtils.java deleted file mode 100644 index 0d8d9b2..0000000 --- a/src/main/java/com/bfd/qanda/utils/OtherUtils.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.bfd.qanda.utils; - -import java.security.MessageDigest; - -/** - * 其他工具类 - * @author jian.mao - * @date 2023年9月19日 - * @description - */ -public class OtherUtils { - - - - public static String getMd5(String string) { - try { - MessageDigest md5 = MessageDigest.getInstance("MD5"); - byte[] bs = md5.digest(string.getBytes("UTF-8")); - StringBuilder sb = new StringBuilder(40); - for (byte x : bs) { - if ((x & 0xff) >> 4 == 0) { - sb.append("0").append(Integer.toHexString(x & 0xff)); - } else { - sb.append(Integer.toHexString(x & 0xff)); - } - } - return sb.toString(); - } catch (Exception e) { - - return "nceaform" + System.currentTimeMillis(); - } - } -} diff --git a/src/main/java/com/bfd/qanda/utils/QueueUtil.java b/src/main/java/com/bfd/qanda/utils/QueueUtil.java deleted file mode 100644 index 5796c2a..0000000 --- a/src/main/java/com/bfd/qanda/utils/QueueUtil.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.bfd.qanda.utils; - -import java.util.Map; -import java.util.concurrent.LinkedBlockingDeque; - -/** - * @author:jinming - * @className:QueueUtil - * @version:1.0 - * @description: - * @Date:2023/7/13 15:00 - */ -public class QueueUtil { - - public static LinkedBlockingDeque> taskQueue = new LinkedBlockingDeque>(); - - public static LinkedBlockingDeque sendQueue = new LinkedBlockingDeque(); -} \ No newline at end of file diff --git a/src/main/java/com/bfd/qanda/utils/SpringBootKafka.java b/src/main/java/com/bfd/qanda/utils/SpringBootKafka.java deleted file mode 100644 index be8ad73..0000000 --- a/src/main/java/com/bfd/qanda/utils/SpringBootKafka.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.bfd.qanda.utils; - -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.support.SendResult; -import org.springframework.stereotype.Component; -import org.springframework.util.concurrent.ListenableFuture; -import org.springframework.util.concurrent.ListenableFutureCallback; - -/** - * @PROJECT_NAME: companybusinesscrawl - * @DESCRIPTION:SpringBootKafka 工具类 - * @AUTHOR: ying.zhao - * @DATE: 2023/4/6 11:09 - */ -@Slf4j -@Component -public class SpringBootKafka { - @Autowired - private KafkaTemplate kafkaTemplate; - /** - * 自定义topicKafkaTemplate - */ - /** - * public static final String TOPIC = "companyBussTest"; - **/ - public void send(String topic, String message) { - //发送消息 - ListenableFuture> future = kafkaTemplate.send(topic, message); - future.addCallback(new ListenableFutureCallback>() { - @Override - public void onFailure(Throwable throwable) { - //发送失败的处理 - log.info(topic + " - 生产者 发送消息失败:" + throwable.getMessage()); - } - - @Override - public void onSuccess(SendResult stringObjectSendResult) { - //成功的处理 - log.info(topic + " - 生产者 发送消息成功" ); - } - }); - } -} diff --git a/src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java b/src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java deleted file mode 100644 index 8771354..0000000 --- a/src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.bfd.qanda.utils; - -import java.io.PrintWriter; -import java.io.StringWriter; - -/** - * @author jian.mao - * @date 2023年3月22日 - * @description - */ -public class ThrowMessageUtil { - - /** - * 获取异常信息 - * @param t - * @return - */ - public static String getErrmessage(Throwable t){ - StringWriter stringWriter=new StringWriter(); - t.printStackTrace(new PrintWriter(stringWriter,true)); - return stringWriter.getBuffer().toString(); - } -}