maojian
/
appliction_manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

冲突解决

master
55007 6 months ago
parent
d8321c29db
commit
5c5c2affbc
23 changed files with 18 additions and 2348 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 12
      .settings/org.eclipse.core.resources.prefs
  2. 141
      pom.xml
  3. 24
      src/main/java/com/bfd/qanda/Application.java
  4. 35
      src/main/java/com/bfd/qanda/cache/ConfigCache.java
  5. 65
      src/main/java/com/bfd/qanda/cache/LocalCache.java
  6. 41
      src/main/java/com/bfd/qanda/controller/QandaController.java
  7. 152
      src/main/java/com/bfd/qanda/entity/Constants.java
  8. 128
      src/main/java/com/bfd/qanda/handler/MainHandler.java
  9. 193
      src/main/java/com/bfd/qanda/process/QandaPorcess.java
  10. 17
      src/main/java/com/bfd/qanda/service/QandaService.java
  11. 58
      src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java
  12. 93
      src/main/java/com/bfd/qanda/token/TokenManagerV3.java
  13. 48
      src/main/java/com/bfd/qanda/utils/DataUtil.java
  14. 177
      src/main/java/com/bfd/qanda/utils/DateUtil.java
  15. 914
      src/main/java/com/bfd/qanda/utils/DownLoadUtil.java
  16. 27
      src/main/java/com/bfd/qanda/utils/EncryptionUtil.java
  17. 36
      src/main/java/com/bfd/qanda/utils/FileUtil.java
  18. 53
      src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java
  19. 32
      src/main/java/com/bfd/qanda/utils/JsonUtil.java
  20. 33
      src/main/java/com/bfd/qanda/utils/OtherUtils.java
  21. 18
      src/main/java/com/bfd/qanda/utils/QueueUtil.java
  22. 46
      src/main/java/com/bfd/qanda/utils/SpringBootKafka.java
  23. 23
      src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java

12
.settings/org.eclipse.core.resources.prefs
View File

@ -1,13 +1,7 @@
<<<<<<< HEAD
=\=\=\=\=\=\=
<<<<<<<=HEAD
>>>>>>>=branch 'master' of http\://82.156.111.58\:3000/maojian/appliction_manager.git
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding/<project>=UTF-8
=======
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding/<project>=UTF-8
>>>>>>> branch 'master' of http://82.156.111.58:3000/maojian/appliction_manager.git

141
pom.xml
View File

@ -10,10 +10,10 @@
<version>2.2.4.RELEASE</version>
</parent>
<groupId>com.bfd</groupId>
<artifactId>bigmodel</artifactId>
<artifactId>appliction_manager</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>bigmodel</name>
<name>appliction_manager</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
@ -30,142 +30,31 @@
<version>4.11</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/de.codecentric/spring-boot-admin-starter-server -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/de.codecentric/spring-boot-admin-starter-client -->
<dependency>
<groupId>de.codecentric</groupId>
<artifactId>spring-boot-admin-starter-client</artifactId>
<version>2.2.4</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.8</version>
<groupId>de.codecentric</groupId>
<artifactId>spring-boot-admin-starter-server</artifactId>
<version>2.2.4</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-mail -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-test</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework/spring-test -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>5.0.10.RELEASE</version>
<scope>test</scope>
<artifactId>spring-boot-starter-mail</artifactId>
<version>2.2.4.RELEASE</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.projectlombok/lombok -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>1.4</version>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.16</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>2.0.17</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.mchange/c3p0 -->
<dependency>
<groupId>com.mchange</groupId>
<artifactId>c3p0</artifactId>
<version>0.9.5.5</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.29</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.9.3</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.jetbrains.kotlin/kotlin-reflect -->
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-reflect</artifactId>
<version>1.6.21</version>
<scope>runtime</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.8.1</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<!-- <dependency> <groupId>org.springframework.kafka</groupId> <artifactId>spring-kafka</artifactId>
</dependency> -->
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.5</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/p6spy/p6spy -->
<dependency>
<groupId>p6spy</groupId>
<artifactId>p6spy</artifactId>
<version>3.9.0</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>6.0.0</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>6.0.0</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>transport-netty4-client</artifactId>
<version>6.0.0</version>
</dependency>
<dependency>
<groupId>com.auth0</groupId>
<artifactId>java-jwt</artifactId>
<version>4.2.2</version>
</dependency>
<dependency>
<groupId>com.bfd.util</groupId>
<artifactId>pauseTool</artifactId>
<version>1.0</version>
</dependency>
</dependencies>
<build>
@ -217,7 +106,7 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<mainClass>com.bfd.qanda.Application</mainClass>
<mainClass>com.bfd.manager.Application</mainClass>
<layout>ZIP</layout>
<includes>
<include>

24
src/main/java/com/bfd/qanda/Application.java
View File

@ -1,24 +0,0 @@
package com.bfd.qanda;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.scheduling.annotation.EnableScheduling;
/**
* 主入口
*
* @author jian.mao
* @date 2023年7月4日
* @description
*/
@SpringBootApplication
@EnableKafka
public class Application {
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}
}

35
src/main/java/com/bfd/qanda/cache/ConfigCache.java
View File

@ -1,35 +0,0 @@
package com.bfd.qanda.cache;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
import java.util.concurrent.LinkedBlockingDeque;
/**
* @author jian.mao
* @date 2022年11月11日
* @description 静态变量类
*/
@Slf4j
public class ConfigCache {
/**启动条件**/
public static boolean isStart = true;
/*****任务队列*****/
public static LinkedBlockingDeque<Map<String, Object>> taskQueue = new LinkedBlockingDeque<Map<String,Object>>();
/**
* 队列录入任务
* @param queue
* @param task
*/
public static void putQueue(LinkedBlockingDeque<Map<String, Object>> queue,Map<String, Object> task){
//next app 写入队列准备调出
try {
queue.put(task);
} catch (InterruptedException e) {
log.error("队列写入data失败---");
}
}
}

65
src/main/java/com/bfd/qanda/cache/LocalCache.java
View File

@ -1,65 +0,0 @@
package com.bfd.qanda.cache;
import java.util.Calendar;
import java.util.Date;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* token 缓存
* @author jian.mao
* @date 2023年11月8日
* @description
*/
@Slf4j
public class LocalCache {
private static final ConcurrentMap<String, Value> CACHE = new ConcurrentHashMap<>(64);
private LocalCache() {
}
public static LocalCache getInstance() {
return Inner.LOCAL_CACHE;
}
public static String get(String key) {
Value v = LocalCache.CACHE.get(key);
if (v == null || new Date().after(v.end)) {
return "";
}
log.debug("get key:{},time left:{}s", key, (v.end.getTime() - new Date().getTime()) / 1000);
return v.value;
}
public static void set(String key, String value, int expire, TimeUnit timeUnit) {
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.SECOND, (int) timeUnit.toSeconds(expire));
Value v = new Value(value, calendar.getTime());
log.debug("put key:{}, expire time:{} ", key, calendar.getTime());
LocalCache.CACHE.put(key, v);
}
private static class Value {
String value;
Date end;
public Value(String value, Date time) {
this.value = value;
this.end = time;
}
}
private static class Inner {
private static final LocalCache LOCAL_CACHE = new LocalCache();
}
}

41
src/main/java/com/bfd/qanda/controller/QandaController.java
View File

@ -1,41 +0,0 @@
package com.bfd.qanda.controller;
import javax.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import com.bfd.qanda.service.QandaService;
/**
* @author jian.mao
* @date 2023年11月8日
* @description
*/
@Controller
@RequestMapping("/qanda")
@Slf4j
public class QandaController {
@Resource
private QandaService qandaService;
@PostMapping("/putQuestion")
@ResponseBody
public String putQuestion(@RequestBody String dataJson){
String response = qandaService.putQuestion(dataJson);
return response;
}
@RequestMapping(value = "/hello", method = RequestMethod.GET)
@ResponseBody
public String hello(String param, String token) {
return "123";
}
}

152
src/main/java/com/bfd/qanda/entity/Constants.java
View File

@ -1,152 +0,0 @@
package com.bfd.qanda.entity;
/**
* 常量实体类
* @author jian.mao
* @date 2022年11月15日
* @description
*/
public class Constants {
/*************************蓝图常量key名称*********************************/
public final static String SCHEDULING = "scheduling";
public final static String TYPE = "type";
public final static String INTERVAL = "interval";
public final static String CREATED = "created";
public final static String LAST_EDIT = "last_edit";
public final static String BLUEPRINT_ID = "blueprint_id";
public final static String BLUEPRINTID = "blueprintId";
public final static String BLUEPRINT_NAME = "name";
public final static String SCENARIO = "scenario";
public final static String AUTOCOMMITTRIGGERLAST = "autoCommitTriggerLast";
public final static String FRESHVARIABLES = "freshVariables";
public final static String AUTOCOMMIT = "autoCommit";
public final static String MAXERRORS = "maxErrors";
public final static String DATALOSS = "dataloss";
public final static String POSITION = "position";
public final static String SCENES_ID = "scenes_id";
public final static String SCENESID = "scenesId";
public final static String MULTI_BRANCH = "multi_branch";
public final static String SINGLE = "single";
/**已重试次数**/
public final static String ERROR_TIME = "error_time";
public final static String PREVIOUS_RESULT = "previous_result";
/****数据id*****/
public final static String BUSINESSKEY = "businessKey";
/*************************metadata常量key名称*********************************/
public final static String LABEL_COL = "label_col";
public final static String LABEL = "label";
public final static String USER = "user";
public final static String ADMIN = "admin";
public final static String ADDRESS = "address";
public final static String DATASOURCE = "datasource";
public final static String INDEX = "index";
/*************************app常量key名称*********************************/
public final static String APPS = "apps";
public final static String TRANSFER_ID = "transfer_id";
public final static String MODULE = "module";
public final static String VERSION = "version";
public final static String METADATA = "metadata";
public final static String APP_NAME = "name";
public final static String DESCRIBE = "describe";
public final static String NEXT_APP_ID = "next_app_id";
public final static String EDGE_ID = "edge_id";
public final static String START_ID = "start_id";
public final static String END_ID = "end_id";
public final static String WAIT_CONDITION = "wait_condition";
public final static String START_TAG = "start_tag";
/*************************module类型*********************************/
public final static String FILE = "file";
public final static String OCR = "OCR";
public final static String FILTER = "Filter";
public final static String CHATGPT = "ChatGPT";
public final static String MYSQL = "mysql";
/*************************other类型*********************************/
public final static String UNDERLINE = "_";
public final static String RESULT_TOPIC = null;
public static final String EMPTY = "";
public static final String HTTP = "http";
public static final String REQUEST_ERROR_MESSAGE = "Download failed error is";
public static final String REQUEST_RESULT = "result";
public static final String REQUEST_RESULT_RESULTS = "results";
public static final String MAP_TYPE = "Map";
public static final String LIST_TYPE = "List";
public static final String STRING_TYPE = "String";
public static final String DOCUMENT_TYPE = "doc";
public static final String FILTER_ZH = "过滤器";
public static final String JSON_SELE_SYMBOL = "$.";
public static final String LEFT_BRACKETS = "[";
public static final String RIGTH_BRACKETS = "]";
public static final String TASKTYPE = "taskType";
public static final Integer USER_TYPE = 1;
public static final Integer KEYWORD_TYPE = 0;
public static final Integer DETAIL_TYPE = 2;
public static final String CID = "cid";
public static final String SITETYPE = "siteType";
public static final Integer DEFULT_SUBJECTID = 304864;
public static final Integer DEFULT_CRAWLCYCLICITYTIME = 1440;
public static final String CRAWLENDTIME = "crawlEndTime";
public static final String CRAWLSTARTTIME = "crawlStartTime";
public static final String CRAWLPAGETYPES = "crawlPageTypes";
public static final String APPID = "113ic";
public static final String APP_ID = "appId";
public final static String ID = "id";
public static final Integer DEFULT_CRAWLPERIODHOUR = 24;
public static final String CREATEUSERID = "662015832180933762";
public static final String CRAWL_ADD_URL = "https://caiji.percent.cn/api/crawl/remote/task/save";
public static final String CRAWLKEYWORD = "crawlKeyword";
public static final String ATTACHTAG = "attachTag";
public static final String ATTACHTAG_VALUE = "analyze";
public static final String KEYWORD = "keyword";
public static final String SITEID = "siteId";
public static final String RESULTS = "results";
public static final String RESULT = "result";
public static final String CRAWLDATAFLAG = "crawlDataFlag";
public static final String CRAWLDATAFLAG_PREFIX = "\"crawlDataFlag\":\"keyword:";
public static final String TID = "tid";
public static final Long TIME_OUT = 1800000L;
public static final String ATTR = "attr";
public static final String HASVIDEO = "hasVideo";
public static final String CRAWL_END_MARK = "crawl_end_mark";
public static final String CRAWL_END_MESSAGE = "crawl_end_message";
public static final String CRAWL_END_MESSAGE_VALUE = "数据采集完成";
public static final String SUBJECTID = "subjectId";
public static final String TASKID = "taskId";
public static final int SUCCESS_CODE = 200;
public static final String WEB_URL_SUFFIX = "/api/aogeo/api/cda/caiji/status";
public static final String STATUS = "status";
/************************redis*************************************/
public static final String LOCK_KEY = "myLock";
public static final long LOCK_EXPIRE_TIME = 300000;
/************************应用参数*************************************/
public static final String CODE = "code";
public static final String MESSAGE = "message";
public static final String INPUT = "input";
public static final String OUTPUT = "output";
public static final String FORM = "form";
public static final String FIELD = "field";
public static final String VALUE = "value";
public static final String DATA = "data";
public static final String COLON_EN = ":";
/******************************admin*******************************/
public static final String AUTHORIZATION = "authorization";
public static final String TEMPERATURE = "temperature";
public static final String FIELDTYPE = "fieldType";
public static final String TOP_P = "top_p";
public static final String MODEL = "model";
public static final String PROMPT = "prompt";
public static final String CHATGLM_TURBO_URL = "https://open.bigmodel.cn/api/paas/v4/chat/completions";
public static final String CHOICES = "choices";
public static final String CONTENT = "content";
}

128
src/main/java/com/bfd/qanda/handler/MainHandler.java
View File

@ -1,128 +0,0 @@
package com.bfd.qanda.handler;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingDeque;
import javax.annotation.Resource;
import com.bfd.util.PauseTool;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.core.annotation.Order;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSONObject;
import com.bfd.qanda.cache.ConfigCache;
import com.bfd.qanda.process.QandaPorcess;
import com.bfd.qanda.utils.FileUtil;
/**
* 启动处理入口
*
* @author jian.mao
* @date 2023年11月3日
* @description
*/
@Component
@Order(value = 1)
@Slf4j
public class MainHandler implements ApplicationRunner {
@Value("${task.task-queue-path}")
private String taskPath;
@Value("${task.thread-num}")
private Integer threadNum;
@Resource
QandaPorcess qandaPorcess;
@Resource
private StringRedisTemplate stringRedisTemplate;
@Value("${zookeeper.connection-string}")
private String connectionString;
@Value("${zookeeper.publish-node}")
private String nodePath;
@Override
public void run(ApplicationArguments args) throws Exception {
PauseTool pauseTool = new PauseTool();
pauseTool.initializeRedisCache(stringRedisTemplate);
pauseTool.setupZookeeperListener(connectionString, nodePath);
for (Integer i = 0; i < threadNum; i++) {
//开启问答处理线程
new Thread(qandaPorcess).start();
log.info("开启问答处理线程-----");
}
//停止处理
waitDown();
//启动加载缓存任务
readTask(taskPath, ConfigCache.taskQueue);
}
@SuppressWarnings("unchecked")
public static void readTask(String path, LinkedBlockingDeque<Map<String, Object>> queue) {
File file = new File(path);
if (file.exists()) {
List<String> tasks = null;
try {
tasks = FileUtils.readLines(file, "UTF-8");
} catch (IOException e) {
e.printStackTrace();
}
for (String taskStr : tasks) {
Map<String, Object> task = JSONObject.parseObject(taskStr);
try {
queue.put(task);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
file.delete();
}
}
/**
* 结束触发钩子
*/
public void waitDown() {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
// 停止线程
ConfigCache.isStart = false;
log.info("stop-------");
writeTsskToFile();
}
});
}
/**
* 任务持久化到硬盘
*/
public void writeTsskToFile() {
while (true) {
if (ConfigCache.taskQueue.size() > 0) {
try {
Map<String, Object> task = ConfigCache.taskQueue.take();
FileUtil.writeFile(taskPath, JSONObject.toJSONString(task));
} catch (InterruptedException e) {
e.printStackTrace();
}
} else {
log.info("taskQueue write is file end");
break;
}
}
}
}

193
src/main/java/com/bfd/qanda/process/QandaPorcess.java
View File

@ -1,193 +0,0 @@
package com.bfd.qanda.process;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.bfd.qanda.utils.GPTResultParseUtil;
import com.bfd.util.PauseTool;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSONObject;
import com.bfd.qanda.cache.ConfigCache;
import com.bfd.qanda.entity.Constants;
import com.bfd.qanda.token.TokenManagerV3;
import com.bfd.qanda.utils.DataUtil;
import com.bfd.qanda.utils.DownLoadUtil;
import com.bfd.qanda.utils.SpringBootKafka;
/**
* @author jian.mao
* @date 2023年11月8日
* @description
*/
@Component
@Slf4j
public class QandaPorcess implements Runnable {
private static final int CHAT_TYPE_ONE = 1;
@Autowired
private SpringBootKafka springBootKafka;
@Value("${customize-kafka.producer.topic}")
private String topic;
@Override
public void run() {
while (ConfigCache.isStart) {
Map<String, Object> task = null;
try {
task = ConfigCache.taskQueue.take();
//结果接收
Map<String, Object> result = new HashMap<String, Object>(16);
//结果内容
StringBuffer chatContent = new StringBuffer();
//输入配置
Map<String, Object> input = (Map<String, Object>) task.get(Constants.INPUT);
//输出配置
Map<String, Object> output = (Map<String, Object>) task.get(Constants.OUTPUT);
//数据源
Map<String, Object> data = (Map<String, Object>) task.get(Constants.DATA);
int scenesId = (int) task.get(Constants.SCENES_ID);
int version = (int) task.get(Constants.VERSION);
String pauseKey = scenesId + "_" + version;
if (!PauseTool.CACHE.containsKey(pauseKey)) {
log.info("流程:{}的版本:{}已失效,任务跳过", scenesId, version);
continue;
}
//请求身份校验 api-key
String authorization = (String) input.get(Constants.AUTHORIZATION);
//fieldType：自定义输出字段： 0 关闭，1-开启，如果开启则拼接form到output里（如果关闭，则取默认的output拼接）
int fieldType = (int) input.get(Constants.FIELDTYPE);
Float temperature = Float.valueOf(input.get(Constants.TEMPERATURE).toString());
Float topP = Float.valueOf(input.get(Constants.TOP_P).toString());
String model = (String) input.get(Constants.MODEL).toString();
List<Map<String, Object>> prompt = (List<Map<String, Object>>) input.get(Constants.PROMPT);
String answerStr = qandaRequest(authorization, model, temperature, topP, prompt, data);
log.info("answerStr:" + answerStr);
Map<String, Object> answer = JSONObject.parseObject(answerStr);
try {
//请求成功，正常解析
List<Map<String, Object>> choices = (List<Map<String, Object>>) answer.get(Constants.CHOICES);
Map<String, Object> message = (Map<String, Object>) choices.get(0).get(Constants.MESSAGE);
chatContent.append(message.get(Constants.CONTENT));
} catch (Exception e) {
log.error("问答接口响应体异常:{}", answerStr, e);
// TODO: handle exception
continue;
}
Map<String, Object> results = new HashMap<String, Object>(16);
results.put(Constants.ID, UUID.randomUUID().toString());
results.put(Constants.CONTENT, chatContent.toString());
if (fieldType != 0) {
results.remove(Constants.CONTENT);
try {
//请求成功，正常解析
List<Map<String, Object>> choices = (List<Map<String, Object>>) answer.get(Constants.CHOICES);
Map<String, Object> message = (Map<String, Object>) choices.get(0).get(Constants.MESSAGE);
String reponseContent = (String) message.get(Constants.CONTENT);
Map<String, Object> stringObjectMap = GPTResultParseUtil.parseGPTResult(output, reponseContent);
results.putAll(stringObjectMap);
} catch (Exception e) {
log.error("问答接口响应体异常:{}", answerStr, e);
// TODO: handle exception
continue;
}
}
results.put("isLast", 1);
result.put(Constants.RESULTS, JSONObject.toJSONString(results));
result.put(Constants.MESSAGE, "成功");
result.put(Constants.STATUS, 1);
task.put(Constants.RESULT, result);
// //发送kafka
springBootKafka.send(topic, JSONObject.toJSONString(task));
log.info("数据流转至下游-------");
} catch (Exception e) {
log.error("问答处理异常，", e);
//结果集
Map<String, Object> results = new HashMap<String, Object>(16);
Map<String, Object> result = new HashMap<String, Object>(16);
//遍历入库返回结果，拼接响应内容
results.put("isLast", 1);
results.put("content", e.getMessage());
result.put(Constants.RESULTS, JSONObject.toJSONString(results));
result.put(Constants.MESSAGE, "异常");
result.put(Constants.STATUS, 2);
task.put(Constants.RESULT, result);
//发送kafka
springBootKafka.send(topic, JSONObject.toJSONString(task));
log.info("数据流转至下游-------");
}
}
}
/**
* 问题请求
*
* @param authorization
* @param temperature
* @param topP
* @param prompt
* @return
*/
private String qandaRequest(String authorization, String model, Float temperature, Float topP, List<Map<String, Object>> prompt, Map<String, Object> data) {
//新建聊天话术
StringBuffer chatContent = new StringBuffer();
for (Map<String, Object> map : prompt) {
if (Integer.valueOf(map.get(Constants.TYPE).toString()) == CHAT_TYPE_ONE) {
chatContent.append(map.get(Constants.VALUE));
} else {
String jsonPath = (String) map.get(Constants.VALUE);
chatContent.append(DataUtil.getValue(jsonPath, data).toString());
}
}
//拆分authorization
String[] apiCombination = authorization.split("\\.");
//获取token
String token = TokenManagerV3.getToken(apiCombination[0], apiCombination[1]);
StringBuffer encryptionAuthorization = new StringBuffer("Bearer ");
encryptionAuthorization.append(token);
Map<String, Object> headers = new HashMap<String, Object>(16);
Map<String, Object> params = new HashMap<String, Object>(16);
List<Map<String, Object>> prompts = buildParam(chatContent.toString(), data);
params.put("model", model);
params.put("messages", prompts);
params.put("temperature", temperature);
params.put("top_p", topP);
headers.put("Authorization", encryptionAuthorization.toString());
headers.put("Content-Type", "application/json");
String html = DownLoadUtil.doPost(Constants.CHATGLM_TURBO_URL, JSONObject.toJSONString(params), headers);
return html;
}
/**
* 参数构建
*
* @param prompt
* @param data
* @return
*/
private List<Map<String, Object>> buildParam(String prompt, Map<String, Object> data) {
//聊天体
List<Map<String, Object>> prompts = new ArrayList<Map<String, Object>>();
log.info("question---{}", prompt);
Map<String, Object> chat1 = new HashMap<String, Object>(16);
chat1.put("role", "user");
chat1.put("content", prompt);
prompts.add(chat1);
return prompts;
}
}

17
src/main/java/com/bfd/qanda/service/QandaService.java
View File

@ -1,17 +0,0 @@
package com.bfd.qanda.service;
/**
* 问答业务层
* @author jian.mao
* @date 2023年11月8日
* @description
*/
public interface QandaService {
/**
* 问答
* @param dataJson 接口参数
* @return
*/
public String putQuestion(String dataJson);
}

58
src/main/java/com/bfd/qanda/service/impl/QandaServiceImpl.java
View File

@ -1,58 +0,0 @@
package com.bfd.qanda.service.impl;
import java.util.HashMap;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSONObject;
import com.bfd.qanda.cache.ConfigCache;
import com.bfd.qanda.entity.Constants;
import com.bfd.qanda.service.QandaService;
/**
* 业务层实现类
*
* @author jian.mao
* @date 2023年11月8日
* @description
*/
@Service
@Slf4j
public class QandaServiceImpl implements QandaService {
@Override
public String putQuestion(String dataJson) {
String trace = "trace";
Map<String, Object> response = new HashMap<>(16);
int code = 200;
String message = "success";
Map<String, Object> task = null;
try {
task = JSONObject.parseObject(dataJson);
} catch (Exception e) {
log.error("参数结构不合法，", e);
code = 100010;
message = "参数不合法";
}
// 写入队列
try {
if (task.containsKey(trace) && (Boolean) task.get(trace)==true) {
log.info("测试流程,插入队首");
ConfigCache.taskQueue.putFirst(task);
}else {
ConfigCache.taskQueue.put(task);
}
} catch (InterruptedException e) {
log.error("任务写入等待队列异常，", e);
code = 100011;
message = "任务写入等待队列失败";
}
response.put(Constants.CODE, code);
response.put(Constants.MESSAGE, message);
return JSONObject.toJSONString(response);
}
}

93
src/main/java/com/bfd/qanda/token/TokenManagerV3.java
View File

@ -1,93 +0,0 @@
package com.bfd.qanda.token;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import com.auth0.jwt.JWT;
import com.auth0.jwt.algorithms.Algorithm;
import com.bfd.qanda.cache.LocalCache;
import com.bfd.qanda.entity.Constants;
/**
* token管理类
* @author jian.mao
* @date 2023年11月8日
* @description
*/
@Slf4j
public class TokenManagerV3 {
private static final String TOKENV3KEYPREFIX = "zhipu_oapi_token_v3";
private static final String ALG = "HS256";
/***默认过期时间30分钟****/
private static final int EXPIREMILLIS = 30 * 60 * 1000;
/**
* token 获取方法
* @param apiKey 用户id
* @param apiSecret 生成key
* @return
*/
public static String getToken(String apiKey,String apiSecret) {
String tokenCacheKey = genTokenCacheKey(apiKey);
String cacheToken = LocalCache.get(tokenCacheKey);
if(cacheToken != null && !cacheToken.equals(Constants.EMPTY)){
return cacheToken;
}
String newToken = createJwt(apiKey,apiSecret);
LocalCache.set(tokenCacheKey, newToken, EXPIREMILLIS, TimeUnit.MILLISECONDS);
return newToken;
}
/**
* token 计算
* @param apiKey
* @param apiSecret
* @return
*/
private static String createJwt(String apiKey,String apiSecret) {
Algorithm alg;
String hs = "HS256";
if (ALG.equals(hs)) {
//alg = Algorithm.HMAC256(config.getApiSecret());
try {
alg = Algorithm.HMAC256(apiSecret.getBytes("utf-8"));
} catch (Exception e) {
e.printStackTrace();
return null;
}
} else {
// 目前仅支持HS256
log.error("algorithm: %s not supported", ALG);
return null;
}
Map<String, Object> payload = new HashMap<>(16);
payload.put("api_key", apiKey);
payload.put("exp", System.currentTimeMillis() + EXPIREMILLIS);
payload.put("timestamp", Calendar.getInstance().getTimeInMillis());
Map<String, Object> headerClaims = new HashMap<>(16);
headerClaims.put("alg", "HS256");
headerClaims.put("sign_type", "SIGN");
String token = JWT.create().withPayload(payload).withHeader(headerClaims).sign(alg);
return token;
}
/**
* 获取缓存key
* @param apiKey
* @return
*/
private static String genTokenCacheKey(String apiKey) {
return String.format("%s-%s", TOKENV3KEYPREFIX, apiKey);
}
public static void main(String[] args) {
String apiKey ="53dbef19a3252a292890408707190f9f";
String apiSecret="5An47w6PngrTFw19";
System.out.println(getToken(apiKey,apiSecret));
}
}

48
src/main/java/com/bfd/qanda/utils/DataUtil.java
View File

@ -1,48 +0,0 @@
package com.bfd.qanda.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONPath;
import java.util.Map;
/**
* @author:jinming
* @className:DataUtil
* @version:1.0
* @description: 获取dataValue的值
* @Date:2023/11/1 9:54
*/
public class DataUtil {
/**
*
* @param key 传入的key
* @param dataMap 数据map
* @return 根据传入的参数进行判断解析，返回正确的dataValue
*/
public static Object getValue(String key, Map dataMap) {
Object dataValue;
String isJson = "#json#";
if (key.contains(isJson)) {
//进行第一次拆分，获取#json#前面的部分
String[] keySplit = key.split(isJson);
String firstDataKey = keySplit[0];
String[] firstDataKeySplit = firstDataKey.split(":");
//取出前半部分对应的JSON数据并转换为JSONObject
String dataJson = (String) dataMap.get(firstDataKeySplit[0]);
JSONObject dataJsonObject = JSON.parseObject(dataJson);
//根据key的后半部分取出对应JSONObject中的值
String firstDataKeyJson = (String) JSONPath.eval(dataJsonObject, firstDataKeySplit[1]);
String secDataKey = keySplit[1];
JSONObject firstDataJsonObject = JSON.parseObject(firstDataKeyJson);
dataValue = JSONPath.eval(firstDataJsonObject, secDataKey);
return dataValue;
}
String[] keySplit = key.split(":");
String jsonPath = keySplit[1];
String dataJson = (String) dataMap.get(keySplit[0]);
JSONObject dataJsonObject = JSON.parseObject(dataJson);
dataValue = JSONPath.eval(dataJsonObject, jsonPath);
return dataValue;
}
}

177
src/main/java/com/bfd/qanda/utils/DateUtil.java
View File

@ -1,177 +0,0 @@
package com.bfd.qanda.utils;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import lombok.extern.slf4j.Slf4j;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
/**
* 日期工具类
*
* @author jian.mao
* @date 2022年11月15日
* @description
*/
@Slf4j
public class DateUtil {
/**
* @return
*/
public static String getTimeStrForNow() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH");
return sdf.format(new Date());
}
public static String getTimeStrForDay(long time) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
return sdf.format(new Date(time * 1000));
}
public static String getTimeStrForDay() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
return sdf.format(new Date());
}
public static String getDateTime() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String time = sdf.format(new Date());
return time;
}
public static String getDateTime(Long timestap) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String time = sdf.format(new Date(timestap));
return time;
}
public static String getDate(Long timestap) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
String time = sdf.format(new Date(timestap));
return time;
}
public static String getDateTimeForMonth() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM");
String time = sdf.format(new Date());
return time;
}
/**
* 休眠
*
* @param millis 毫秒
*/
public static void sleep(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/**
* 1. @Description:时间戳转时间
* 2. @Author: ying.zhao
* 3. @Date: 2023/3/28
*/
public static String timestampToDate(String time) {
int thirteen = 13;
int ten = 10;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// if (time.length() == thirteen) {
if (time.length() > ten) {
return sdf.format(new Date(Long.parseLong(time)));
} else {
return sdf.format(new Date(Integer.parseInt(time) * 1000L));
}
}
public static String parseCreated(String jsonTime){
String formattedDateTime = getDateTime();
try {
// 使用fastjson解析JSON数据
JSONObject jsonObject = JSON.parseObject(jsonTime);
// 获取日期和时间的值
JSONObject dateObject = jsonObject.getJSONObject("date");
int day = dateObject.getIntValue("day");
int month = dateObject.getIntValue("month");
int year = dateObject.getIntValue("year");
JSONObject timeObject = jsonObject.getJSONObject("time");
int hour = timeObject.getIntValue("hour");
int minute = timeObject.getIntValue("minute");
int second = timeObject.getIntValue("second");
// 创建LocalDateTime对象
LocalDateTime dateTime = LocalDateTime.of(year, month, day, hour, minute, second);
// 定义日期时间格式化器
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
// 格式化日期时间
formattedDateTime = dateTime.format(formatter);
} catch (Exception e) {
log.info("日期转换失败：{}",e);
}
return formattedDateTime;
}
/**
* 字符串转换日期
* @param format
* @param date
* @return
*/
public static Date strToDate(String format,String date){
SimpleDateFormat sdf = new SimpleDateFormat(format);
if (date == null || date.equals("")){
return new Date();
}else{
Date ru = null;
try {
ru = sdf.parse(date);
} catch (ParseException e) {
e.printStackTrace();
}
return ru;
}
}
/**
* 日期格式话
* @param format 日期格式
* @param dater 要转换的日期,默认当前时间
* @return
*/
public static String FormatDate(String format,Date date){
String fromatDate = null;
SimpleDateFormat sdf = new SimpleDateFormat(format);
if (date == null){
fromatDate = sdf.format(new Date());
}else{
fromatDate = sdf.format(date);
}
return fromatDate;
}
public static void main(String[] args) {
String time = timestampToDate("955814400000");
System.out.println(time);
}
}

914
src/main/java/com/bfd/qanda/utils/DownLoadUtil.java
View File

@ -1,914 +0,0 @@
package com.bfd.qanda.utils;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.bfd.qanda.entity.Constants;
/**
* 下载工具类
* @author jian.mao
* @date 2023年9月19日
* @description
*/
public class DownLoadUtil {
private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36";
private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class);
/** 代理服务器(产品官网 www.16yun.cn) **/
final static String PROXYHOST = "u270.40.tp.16yun.cn";
final static Integer PROXYPORT = 6448;
/** 代理验证信息 **/
final static String PROXYUSER = "16HFBVJC";
final static String PROXYPASS = "897944";
private static PoolingHttpClientConnectionManager cm = null;
private static HttpRequestRetryHandler httpRequestRetryHandler = null;
private static HttpHost proxy = null;
private static CredentialsProvider credsProvider = null;
private static RequestConfig reqConfig = null;
static {
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory
.getSocketFactory();
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory
.getSocketFactory();
Registry registry = RegistryBuilder.create().register("http", plainsf)
.register("https", sslsf).build();
cm = new PoolingHttpClientConnectionManager(registry);
cm.setMaxTotal(20);
cm.setDefaultMaxPerRoute(5);
proxy = new HttpHost(PROXYHOST, PROXYPORT, "https");
credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(PROXYUSER, PROXYPASS));
reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000)
.setConnectTimeout(5000).setSocketTimeout(5000)
.setExpectContinueEnabled(false)
.setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build();
}
/**
* 模拟客户端get请求
*
* @param url
* 模拟请求得url
* @param headers
* 头部信息，没有可以不传
* @return
*/
@SafeVarargs
public static String proxyDoGet(String url, Map<String, Object>... headers) {
// 设置超时时间
int timeout = 30;
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(timeout * 1000)
.setConnectionRequestTimeout(timeout * 1000)
.setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(timeout * 1000)
.setTcpNoDelay(true).build();
AuthCache authCache = new BasicAuthCache();
authCache.put(proxy, new BasicScheme());
HttpClientContext localContext = HttpClientContext.create();
localContext.setAuthCache(authCache);
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
CloseableHttpClient httpClient = httpBuilder
.setDefaultSocketConfig(socketConfig)
.setDefaultRequestConfig(config)
.setDefaultCredentialsProvider(credsProvider).build();
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(reqConfig);
if (headers != null && headers.length > 0) {
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpGet.setHeader(key, tempHeaders.get(key).toString());
}
} else {
httpGet.setHeader("Accept",
"application/json, text/javascript, */*; q=0.01");
httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
}
CloseableHttpResponse response = null;
String html = "";
int notFundCode = 404;
int successCode = 200;
try {
response = httpClient.execute(httpGet, localContext);
// 从响应模型中获取响应实体
HttpEntity responseEntity = response.getEntity();
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
if (statusLine.getStatusCode() == successCode) {
if (responseEntity != null) {
html = EntityUtils.toString(responseEntity, "utf-8");
System.out.println("响应内容长度为:"
+ responseEntity.getContentLength());
// 下载结果为空不正常
if (html.equals(Constants.EMPTY)) {
html = "Download failed error is:reslut is null";
}
}
} else if (statusLine.getStatusCode() == notFundCode) {
html = "<h2>页面404，正常结束请求即可</h2>";
} else {
throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:reslut is null";
}finally{
try {
response.close();
httpClient.close();
} catch (Exception e) {
e.printStackTrace();
}
}
return html;
}
public static String httpsslProxyGet(String url, Map<String, Object>... headers) throws Exception {
//采用绕过验证的方式处理https请求
SSLContext sslcontext = createIgnoreVerifySSL();
// 设置协议http和https对应的处理socket链接工厂的对象
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslcontext))
.build();
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
connManager.setMaxTotal(50);
connManager.setDefaultMaxPerRoute(10);
HttpClients.custom().setConnectionManager(connManager);
// 设置超时时间
int timeout = 30;
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(timeout * 1000)
.setConnectionRequestTimeout(timeout * 1000)
.setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(timeout * 1000)
.setTcpNoDelay(true).build();
AuthCache authCache = new BasicAuthCache();
authCache.put(proxy, new BasicScheme());
HttpClientContext localContext = HttpClientContext.create();
localContext.setAuthCache(authCache);
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
CloseableHttpClient httpClient = httpBuilder
.setConnectionManager(connManager)
.setDefaultSocketConfig(socketConfig)
.setDefaultRequestConfig(config)
.setDefaultCredentialsProvider(credsProvider).build();
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(reqConfig);
if (headers != null && headers.length > 0) {
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpGet.setHeader(key, tempHeaders.get(key).toString());
}
} else {
httpGet.setHeader("Accept",
"application/json, text/javascript, */*; q=0.01");
httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
}
CloseableHttpResponse response = null;
String html = "";
int notFundCode = 404;
int successCode = 200;
try {
response = httpClient.execute(httpGet, localContext);
// 从响应模型中获取响应实体
HttpEntity responseEntity = response.getEntity();
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
if (statusLine.getStatusCode() == successCode) {
if (responseEntity != null) {
html = EntityUtils.toString(responseEntity, "utf-8");
System.out.println("响应内容长度为:"
+ responseEntity.getContentLength());
// 下载结果为空不正常
if (html.equals(Constants.EMPTY)) {
html = "Download failed error is:reslut is null";
}
}
} else if (statusLine.getStatusCode() == notFundCode) {
html = "<h2>页面404，正常结束请求即可</h2>";
} else {
throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:reslut is null";
}finally{
try {
response.close();
httpClient.close();
} catch (Exception e) {
e.printStackTrace();
}
}
return html;
}
/**
* json参数方式POST提交
* @param url
* @param params
* @return
*/
public static String doPost(String url, String params, Map<String, Object>... headers){
String strResult = "";
//设置超时时间
int timeout = 60;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(timeout * 1000)
.setTcpNoDelay(true).build();
// AuthCache authCache = new BasicAuthCache();
// authCache.put(proxy, new BasicScheme());
// HttpClientContext localContext = HttpClientContext.create();
// localContext.setAuthCache(authCache);
// 1. 获取默认的client实例
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
httpBuilder.setUserAgent(ua);
HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
// HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
// .setDefaultCredentialsProvider(credsProvider).build();
// 2. 创建httppost实例
HttpPost httpPost = new HttpPost(url);
// httpPost.setConfig(reqConfig);
if (headers != null && headers.length > 0) {
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpPost.setHeader(key, tempHeaders.get(key).toString());
}
} else {
httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
}
HttpResponse resp = null;
try {
httpPost.setEntity(new StringEntity(params,"utf-8"));
resp = client.execute(httpPost);
// resp = client.execute(httpPost,localContext);
StatusLine statusLine = resp.getStatusLine();
System.out.println("响应状态为:" + resp.getStatusLine());
int notFundCode = 404;
int successCode = 200;
if(statusLine.getStatusCode() == successCode){
// 7. 获取响应entity
HttpEntity respEntity = resp.getEntity();
strResult = EntityUtils.toString(respEntity, "UTF-8");
if(strResult.equals(Constants.EMPTY)){
strResult = "Download failed error is:reslut is null";
}
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
return strResult;
}
public static String httpPost(String url,String params) {
String html="";
html = doPost(url,params);
int i = 1;
while(true){
if(html.contains("Download failed error is:")){
log.error("DownLoadUtil------------->download is failure,url is:"+url);
DateUtil.sleep(5000);
i++;
}else{
break;
}
if(i > 5){
break;
}
html = doPost(url,params);
}
return html;
}
/**
* 绕过验证
*
* @return
* @throws NoSuchAlgorithmException
* @throws KeyManagementException
*/
public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
SSLContext sc = SSLContext.getInstance("SSLv3");
// 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
X509TrustManager trustManager = new X509TrustManager() {
@Override
public void checkClientTrusted(
java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
String paramString) throws CertificateException {
}
@Override
public void checkServerTrusted(
java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
String paramString) throws CertificateException {
}
@Override
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
return null;
}
};
sc.init(null, new TrustManager[] { trustManager }, null);
return sc;
}
/**
* 模拟请求
*
* @param url 资源地址
* @param map 参数列表
* @param encoding 编码
* @return
* @throws NoSuchAlgorithmException
* @throws KeyManagementException
* @throws IOException
* @throws ClientProtocolException
*/
public static String httpsslGet(String url,Map<String, Object> ... headers) {
String html="";
CloseableHttpClient client = null;
HttpEntity responseEntity = null;
CloseableHttpResponse response = null;
try {
log.debug("DownLoadUtil------------->设置下载相关信息， start....");
//采用绕过验证的方式处理https请求
SSLContext sslcontext = createIgnoreVerifySSL();
// 设置协议http和https对应的处理socket链接工厂的对象
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslcontext))
.build();
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
connManager.setMaxTotal(50);
connManager.setDefaultMaxPerRoute(10);
HttpClients.custom().setConnectionManager(connManager);
//设置超时时间
int timeout = 30;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(10000)
.setTcpNoDelay(true).build();
// 设置重定向策略
LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy();
//创建自定义的httpclient对象
client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build();
// CloseableHttpClient client = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(url);
if(headers != null && headers.length > 0){
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpGet.setHeader(key,tempHeaders.get(key).toString());
}
}else{
httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
}
log.debug("DownLoadUtil------------->设置下载相关信息， end....");
try {
int notFundCode = 404;
int successCode = 200;
log.debug("DownLoadUtil------------->下载执行，start....");
httpGet.setConfig(config);
response = client.execute(httpGet);
log.debug("DownLoadUtil------------->下载执行，end....");
// 从响应模型中获取响应实体
StatusLine statusLine = response.getStatusLine();
log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is start ....");
System.out.println("响应状态为:" + response.getStatusLine());
responseEntity = response.getEntity();
log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is end ....");
if(statusLine.getStatusCode() == successCode){
if (responseEntity != null) {
html=EntityUtils.toString(responseEntity,"utf-8");
System.out.println("响应内容长度为:" + responseEntity.getContentLength());
}
}else if(statusLine.getStatusCode() == notFundCode){
html = "<h2>页面404，正常结束请求即可</h2>";
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}finally{
try {
responseEntity.getContent().close();
response.close();
client.close();
} catch (Exception e) {
e.printStackTrace();
}
}
return html;
}
public static String httpSSLGet(String url,Map<String, Object> ... headers) {
String html="";
html = httpsslGet(url,headers);
int i = 1;
while(true){
if(html.contains("Download failed error is:")){
log.error("DownLoadUtil------------->download is failure,url is:"+url);
DateUtil.sleep(30000);
i++;
}else{
break;
}
if(i > 5){
break;
}
html = httpsslGet(url,headers);
}
return html;
}
public static String doPostFrom(String url,Map<String, Object> param,Map<String, Object> ... headers){
//设置超时时间
int timeout = 15;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(10000)
.setTcpNoDelay(true).build();
// AuthCache authCache = new BasicAuthCache();
// authCache.put(proxy, new BasicScheme());
// HttpClientContext localContext = HttpClientContext.create();
// localContext.setAuthCache(authCache);
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
httpBuilder.setUserAgent(ua);
// HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
// .setDefaultCredentialsProvider(credsProvider).build();
HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
HttpPost httpPost = new HttpPost(url);
// httpPost.setConfig(reqConfig);
if(headers != null && headers.length > 0){
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpPost.setHeader(key,tempHeaders.get(key).toString());
}
}else{
httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36");
// httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html");
}
// 创建请求参数
List<NameValuePair> list = new LinkedList<>();
for (String key : param.keySet()) {
BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
list.add(param1);
}
// 使用URL实体转换工具
String html="";
try {
UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
httpPost.setEntity(entityParam);
HttpResponse response = httpClient.execute(httpPost);
// HttpResponse response = httpClient.execute(httpPost,localContext);
// 从响应模型中获取响应实体
HttpEntity responseEntity = response.getEntity();
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
int notFundCode = 404;
int successCode = 200;
if(statusLine.getStatusCode() == successCode){
if (responseEntity != null) {
html=EntityUtils.toString(responseEntity,"utf-8");
}
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
return html;
}
public static String httpPostForm(String url,Map<String,Object> params,Map<String, Object> ... headers) {
String html="";
html = doPostFrom(url,params);
int i = 1;
while(true){
if(html.contains("Download failed error is:")){
log.error("DownLoadUtil------------->download is failure,url is:"+url);
DateUtil.sleep(5000);
i++;
}else{
break;
}
if(i > 5){
break;
}
html = doPostFrom(url,params,headers);
}
return html;
}
public static String dosslPost(String url,String params,Map<String, Object> ... headers) {
String html="";
CloseableHttpClient client = null;
HttpEntity responseEntity = null;
CloseableHttpResponse response = null;
try {
//采用绕过验证的方式处理https请求
SSLContext sslcontext = createIgnoreVerifySSL();
// 设置协议http和https对应的处理socket链接工厂的对象
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslcontext))
.build();
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
HttpClients.custom().setConnectionManager(connManager);
//设置超时时间
int timeout = 5;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(10000)
.setTcpNoDelay(true).build();
//创建自定义的httpclient对象
client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
// CloseableHttpClient client = HttpClients.createDefault();
// 2. 创建httppost实例
HttpPost httpPost = new HttpPost(url);
// httpPost.setConfig(reqConfig);
httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
if(headers != null && headers.length > 0){
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpPost.setHeader(key,tempHeaders.get(key).toString());
}
}else{
httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
}
try {
httpPost.setEntity(new StringEntity(params,"utf-8"));
response = client.execute(httpPost);
int notFundCode = 404;
int successCode = 200;
// 从响应模型中获取响应实体
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
responseEntity = response.getEntity();
if(statusLine.getStatusCode() == successCode){
if (responseEntity != null) {
html=EntityUtils.toString(responseEntity,"utf-8");
System.out.println("响应内容长度为:" + responseEntity.getContentLength());
}
}else if(statusLine.getStatusCode() == notFundCode){
html = "<h2>页面404，正常结束请求即可</h2>";
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}finally{
try {
responseEntity.getContent().close();
response.close();
client.close();
} catch (UnsupportedOperationException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
return html;
}
public static String dosslPostForm(String url,Map<String, Object> param,Map<String, Object> ... headers) {
String html="";
try {
//采用绕过验证的方式处理https请求
SSLContext sslcontext = createIgnoreVerifySSL();
// 设置协议http和https对应的处理socket链接工厂的对象
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslcontext))
.build();
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
HttpClients.custom().setConnectionManager(connManager);
//设置超时时间
int timeout = 5;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(10000)
.setTcpNoDelay(true).build();
//创建自定义的httpclient对象
CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
// CloseableHttpClient client = HttpClients.createDefault();
// 2. 创建httppost实例
HttpPost httpPost = new HttpPost(url);
// httpPost.setConfig(reqConfig);
if(headers != null && headers.length > 0){
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpPost.setHeader(key,tempHeaders.get(key).toString());
}
}else{
httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
}
// 创建请求参数
List<NameValuePair> list = new LinkedList<>();
for (String key : param.keySet()) {
BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
list.add(param1);
}
// 使用URL实体转换工具
try {
UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
httpPost.setEntity(entityParam);
HttpResponse response = client.execute(httpPost);
// HttpResponse response = httpClient.execute(httpPost,localContext);
// 从响应模型中获取响应实体
int notFundCode = 404;
int successCode = 200;
HttpEntity responseEntity = response.getEntity();
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
if(statusLine.getStatusCode() == successCode){
if (responseEntity != null) {
html=EntityUtils.toString(responseEntity,"utf-8");
}
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
return html;
}
public static String httpSSLPostForm(String url,Map<String, Object> params,Map<String, Object> ...headers) {
String html="";
try {
html = dosslPostForm(url,params,headers);
} catch (Exception e) {
e.printStackTrace();
// TODO: handle exception
html = "Download failed error is:Exception!";
}
int i = 1;
while(true){
if(html.contains("Download failed error is:")){
log.error("DownLoadUtil------------->download is failure,url is:"+url);
DateUtil.sleep(30000);
i++;
}else{
break;
}
if(i > 5){
break;
}
try {
html = dosslPostForm(url,params,headers);
} catch (Exception e) {
e.printStackTrace();
// TODO: handle exception
html = "Download failed error is:Exception!";
}
}
return html;
}
public static String httpSSLPost(String url,String params,Map<String, Object> ...headers) {
String html="";
try {
html = dosslPost(url,params,headers);
} catch (Throwable e) {
e.printStackTrace();
// TODO: handle exception
html = "Download failed error is:Exception!";
}
int i = 1;
while(true){
if(html.contains("Download failed error is:")){
log.error("DownLoadUtil------------->download is failure,url is:"+url);
DateUtil.sleep(30000);
i++;
}else{
break;
}
if(i > 5){
break;
}
try {
html = dosslPost(url,params,headers);
} catch (Throwable e) {
e.printStackTrace();
// TODO: handle exception
html = "Download failed error is:Exception!";
}
}
return html;
}
/**
* 模拟客户端get请求
* @param url 模拟请求得url
* @param headers 头部信息，没有可以不传
* @return
*/
public static String doGet(String url,Map<String, Object> ... headers){
//设置超时时间
int timeout = 15;
RequestConfig config = RequestConfig.custom().
setConnectTimeout(timeout * 1000).
setConnectionRequestTimeout(timeout * 1000).
setSocketTimeout(timeout * 1000).build();
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(false)
.setSoLinger(1)
.setSoReuseAddress(true)
.setSoTimeout(10000)
.setTcpNoDelay(true).build();
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
httpBuilder.setUserAgent(ua);
HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
HttpGet httpGet = new HttpGet(url);
if(headers != null && headers.length > 0){
Map<String, Object> tempHeaders = headers[0];
for (String key : tempHeaders.keySet()) {
httpGet.setHeader(key,tempHeaders.get(key).toString());
}
}else{
httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
}
String html="";
try {
int notFundCode = 404;
int successCode = 200;
HttpResponse response = httpClient.execute(httpGet);
// 从响应模型中获取响应实体
HttpEntity responseEntity = response.getEntity();
StatusLine statusLine = response.getStatusLine();
System.out.println("响应状态为:" + response.getStatusLine());
if(statusLine.getStatusCode() == successCode){
if (responseEntity != null) {
html=EntityUtils.toString(responseEntity,"utf-8");
if(html.equals("")){
html = "Download failed error is:reslut is null";
}
}
}else if(statusLine.getStatusCode() == notFundCode){
html = "<h2>页面404，正常结束请求即可</h2>";
}else{
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
}
} catch (Exception e) {
e.printStackTrace();
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
}
return html;
}
public static void main(String[] args) throws Exception {
}
}

27
src/main/java/com/bfd/qanda/utils/EncryptionUtil.java
View File

@ -1,27 +0,0 @@
package com.bfd.qanda.utils;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
/**
* @author jian.mao
* @date 2023年3月10日
* @description
*/
public class EncryptionUtil {
public static String md5(String text) {
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(text.getBytes());
byte[] bytes = md.digest();
StringBuilder sb = new StringBuilder();
for (byte b : bytes) {
sb.append(String.format("%02x", b & 0xff));
}
return sb.toString();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
return null;
}
}
}

36
src/main/java/com/bfd/qanda/utils/FileUtil.java
View File

@ -1,36 +0,0 @@
package com.bfd.qanda.utils;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* 文件工具类
* @author jian.mao
* @date 2023年7月14日
* @description
*/
public class FileUtil {
/**
* 数据写入文件
* @param Path 文件路径
* @param result 数据
* @throws IOException
*/
public static void writeFile(String path,String result){
try {
FileWriter fw = new FileWriter(path,true);
fw.write(result+"\n");
fw.flush();
fw.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}

53
src/main/java/com/bfd/qanda/utils/GPTResultParseUtil.java
View File

@ -1,53 +0,0 @@
package com.bfd.qanda.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONException;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author:jinming
* @className:GPTResultParseUtil
* @version:1.0
* @description:
* @Date:2024/6/28 10:11
*/
public class GPTResultParseUtil {
public static Map<String, Object> parseGPTResult(Map<String, Object> output, String gptContent) {
Map<String, Object> jsonResult = new HashMap<>();
try {
// 替换```json, ``` 和 \n
String jsonContent = gptContent.replace("```json", "").replace("```", "").replace("\n", "");
JSONObject jsonGPT = JSON.parseObject(jsonContent);
for (String key : output.keySet()) {
if (jsonGPT.containsKey(key)) {
jsonResult.put(key, jsonGPT.get(key));
}
}
return jsonResult;
} catch (JSONException e) {
try {
// 直接解析失败，使用正则表达式匹配外层的 {}
Pattern pattern = Pattern.compile("\\{.*\\}", Pattern.DOTALL);
Matcher matcher = pattern.matcher(gptContent.replace("\n", ""));
if (matcher.find()) {
JSONObject jsonGPT = JSON.parseObject(matcher.group());
for (String key : output.keySet()) {
if (jsonGPT.containsKey(key)) {
jsonResult.put(key, jsonGPT.get(key));
}
}
return jsonResult;
} else {
return null;
}
} catch (Exception ex) {
ex.printStackTrace();
return null;
}
}
}
}

32
src/main/java/com/bfd/qanda/utils/JsonUtil.java
View File

@ -1,32 +0,0 @@
package com.bfd.qanda.utils;
import com.alibaba.fastjson.JSONObject;
import com.bfd.qanda.entity.Constants;
/**
* json工具
* @author jian.mao
* @date 2023年7月10日
* @description
*/
public class JsonUtil {
/**
* 校验字符串是list/map/str
* @param jsonString
* @return
*/
public static String checkJsonType(String jsonString) {
try {
JSONObject.parseObject(jsonString);
return Constants.MAP_TYPE;
} catch (Exception e) {
try {
JSONObject.parseArray(jsonString);
return Constants.LIST_TYPE;
} catch (Exception ex) {
return Constants.STRING_TYPE;
}
}
}
}

33
src/main/java/com/bfd/qanda/utils/OtherUtils.java
View File

@ -1,33 +0,0 @@
package com.bfd.qanda.utils;
import java.security.MessageDigest;
/**
* 其他工具类
* @author jian.mao
* @date 2023年9月19日
* @description
*/
public class OtherUtils {
public static String getMd5(String string) {
try {
MessageDigest md5 = MessageDigest.getInstance("MD5");
byte[] bs = md5.digest(string.getBytes("UTF-8"));
StringBuilder sb = new StringBuilder(40);
for (byte x : bs) {
if ((x & 0xff) >> 4 == 0) {
sb.append("0").append(Integer.toHexString(x & 0xff));
} else {
sb.append(Integer.toHexString(x & 0xff));
}
}
return sb.toString();
} catch (Exception e) {
return "nceaform" + System.currentTimeMillis();
}
}
}

18
src/main/java/com/bfd/qanda/utils/QueueUtil.java
View File

@ -1,18 +0,0 @@
package com.bfd.qanda.utils;
import java.util.Map;
import java.util.concurrent.LinkedBlockingDeque;
/**
* @author:jinming
* @className:QueueUtil
* @version:1.0
* @description:
* @Date:2023/7/13 15:00
*/
public class QueueUtil {
public static LinkedBlockingDeque<Map<String, Object>> taskQueue = new LinkedBlockingDeque<Map<String, Object>>();
public static LinkedBlockingDeque<String> sendQueue = new LinkedBlockingDeque<String>();
}

46
src/main/java/com/bfd/qanda/utils/SpringBootKafka.java
View File

@ -1,46 +0,0 @@
package com.bfd.qanda.utils;
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.stereotype.Component;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.ListenableFutureCallback;
/**
* @PROJECT_NAME: companybusinesscrawl
* @DESCRIPTION:SpringBootKafka 工具类
* @AUTHOR: ying.zhao
* @DATE: 2023/4/6 11:09
*/
@Slf4j
@Component
public class SpringBootKafka {
@Autowired
private KafkaTemplate<String, Object> kafkaTemplate;
/**
* 自定义topicKafkaTemplate
*/
/**
* public static final String TOPIC = "companyBussTest";
**/
public void send(String topic, String message) {
//发送消息
ListenableFuture<SendResult<String, Object>> future = kafkaTemplate.send(topic, message);
future.addCallback(new ListenableFutureCallback<SendResult<String, Object>>() {
@Override
public void onFailure(Throwable throwable) {
//发送失败的处理
log.info(topic + " - 生产者 发送消息失败：" + throwable.getMessage());
}
@Override
public void onSuccess(SendResult<String, Object> stringObjectSendResult) {
//成功的处理
log.info(topic + " - 生产者 发送消息成功" );
}
});
}
}

23
src/main/java/com/bfd/qanda/utils/ThrowMessageUtil.java
View File

@ -1,23 +0,0 @@
package com.bfd.qanda.utils;
import java.io.PrintWriter;
import java.io.StringWriter;
/**
* @author jian.mao
* @date 2023年3月22日
* @description
*/
public class ThrowMessageUtil {
/**
* 获取异常信息
* @param t
* @return
*/
public static String getErrmessage(Throwable t){
StringWriter stringWriter=new StringWriter();
t.printStackTrace(new PrintWriter(stringWriter,true));
return stringWriter.getBuffer().toString();
}
}
Write Preview
Loading…
Cancel
Save