commit
ed10c95aa6
27 changed files with 2570 additions and 0 deletions
-
40.classpath
-
3.gitignore
-
23.project
-
1README.md
-
228pom.xml
-
24src/main/java/com/bfd/Application.java
-
37src/main/java/com/bfd/cache/ConfigCache.java
-
37src/main/java/com/bfd/controller/ManagementController.java
-
183src/main/java/com/bfd/entity/Constants.java
-
132src/main/java/com/bfd/handler/MainHandler.java
-
74src/main/java/com/bfd/process/ModelSelectProcess.java
-
57src/main/java/com/bfd/process/VideoModelSelectProcess.java
-
25src/main/java/com/bfd/service/ManagementService.java
-
77src/main/java/com/bfd/service/impl/ManagementServiceImpl.java
-
63src/main/java/com/bfd/utils/DataUtil.java
-
177src/main/java/com/bfd/utils/DateUtil.java
-
1007src/main/java/com/bfd/utils/DownLoadUtil.java
-
27src/main/java/com/bfd/utils/EncryptionUtil.java
-
36src/main/java/com/bfd/utils/FileUtil.java
-
32src/main/java/com/bfd/utils/JsonUtil.java
-
65src/main/java/com/bfd/utils/OcrUtil.java
-
33src/main/java/com/bfd/utils/OtherUtils.java
-
18src/main/java/com/bfd/utils/QueueUtil.java
-
23src/main/java/com/bfd/utils/ThrowMessageUtil.java
-
92src/main/resources/application.yml
-
36src/main/resources/logback-spring.xml
-
20src/test/java/com/bfd/AppTest.java
@ -0,0 +1,40 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<classpath> |
|||
<classpathentry kind="src" output="target/classes" path="src/main/java"> |
|||
<attributes> |
|||
<attribute name="optional" value="true"/> |
|||
<attribute name="maven.pomderived" value="true"/> |
|||
</attributes> |
|||
</classpathentry> |
|||
<classpathentry kind="src" output="target/test-classes" path="src/test/java"> |
|||
<attributes> |
|||
<attribute name="optional" value="true"/> |
|||
<attribute name="maven.pomderived" value="true"/> |
|||
<attribute name="test" value="true"/> |
|||
</attributes> |
|||
</classpathentry> |
|||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"> |
|||
<attributes> |
|||
<attribute name="maven.pomderived" value="true"/> |
|||
<attribute name="optional" value="true"/> |
|||
</attributes> |
|||
</classpathentry> |
|||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"> |
|||
<attributes> |
|||
<attribute name="maven.pomderived" value="true"/> |
|||
</attributes> |
|||
</classpathentry> |
|||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"> |
|||
<attributes> |
|||
<attribute name="maven.pomderived" value="true"/> |
|||
</attributes> |
|||
</classpathentry> |
|||
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources"> |
|||
<attributes> |
|||
<attribute name="maven.pomderived" value="true"/> |
|||
<attribute name="test" value="true"/> |
|||
<attribute name="optional" value="true"/> |
|||
</attributes> |
|||
</classpathentry> |
|||
<classpathentry kind="output" path="target/classes"/> |
|||
</classpath> |
@ -0,0 +1,3 @@ |
|||
/logs/ |
|||
/target/ |
|||
.settings/ |
@ -0,0 +1,23 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<projectDescription> |
|||
<name>IntelliModelManager</name> |
|||
<comment></comment> |
|||
<projects> |
|||
</projects> |
|||
<buildSpec> |
|||
<buildCommand> |
|||
<name>org.eclipse.jdt.core.javabuilder</name> |
|||
<arguments> |
|||
</arguments> |
|||
</buildCommand> |
|||
<buildCommand> |
|||
<name>org.eclipse.m2e.core.maven2Builder</name> |
|||
<arguments> |
|||
</arguments> |
|||
</buildCommand> |
|||
</buildSpec> |
|||
<natures> |
|||
<nature>org.eclipse.jdt.core.javanature</nature> |
|||
<nature>org.eclipse.m2e.core.maven2Nature</nature> |
|||
</natures> |
|||
</projectDescription> |
@ -0,0 +1 @@ |
|||
大预言模型管理 |
@ -0,0 +1,228 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
|||
<modelVersion>4.0.0</modelVersion> |
|||
|
|||
<groupId>com.bfd</groupId> |
|||
<artifactId>IntelliModelManager</artifactId> |
|||
<version>0.0.1-SNAPSHOT</version> |
|||
<parent> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter-parent</artifactId> |
|||
<version>2.2.4.RELEASE</version> |
|||
</parent> |
|||
<name>IntelliModelManager</name> |
|||
<!-- FIXME change it to the project's website --> |
|||
<url>http://www.example.com</url> |
|||
|
|||
<properties> |
|||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
|||
<maven.compiler.source>1.8</maven.compiler.source> |
|||
<maven.compiler.target>1.8</maven.compiler.target> |
|||
</properties> |
|||
|
|||
<dependencies> |
|||
<dependency> |
|||
<groupId>junit</groupId> |
|||
<artifactId>junit</artifactId> |
|||
<version>4.11</version> |
|||
<scope>test</scope> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter-web</artifactId> |
|||
</dependency> |
|||
<!-- https://mvnrepository.com/artifact/de.codecentric/spring-boot-admin-starter-client --> |
|||
<dependency> |
|||
<groupId>de.codecentric</groupId> |
|||
<artifactId>spring-boot-admin-starter-client</artifactId> |
|||
<version>2.2.4</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>com.google.code.gson</groupId> |
|||
<artifactId>gson</artifactId> |
|||
<version>2.8.8</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-test</artifactId> |
|||
</dependency> |
|||
<!-- https://mvnrepository.com/artifact/org.springframework/spring-test --> |
|||
<dependency> |
|||
<groupId>org.springframework</groupId> |
|||
<artifactId>spring-test</artifactId> |
|||
<version>5.0.10.RELEASE</version> |
|||
<scope>test</scope> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>commons-io</groupId> |
|||
<artifactId>commons-io</artifactId> |
|||
<version>2.11.0</version> <!-- 根据你的需求选择版本 --> |
|||
</dependency> |
|||
|
|||
<dependency> |
|||
<groupId>com.alibaba</groupId> |
|||
<artifactId>fastjson</artifactId> |
|||
<version>2.0.17</version> |
|||
</dependency> |
|||
<!-- https://mvnrepository.com/artifact/com.mchange/c3p0 --> |
|||
<dependency> |
|||
<groupId>com.mchange</groupId> |
|||
<artifactId>c3p0</artifactId> |
|||
<version>0.9.5.5</version> |
|||
</dependency> |
|||
<!-- https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp --> |
|||
<dependency> |
|||
<groupId>com.squareup.okhttp3</groupId> |
|||
<artifactId>okhttp</artifactId> |
|||
<version>4.9.3</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.apache.httpcomponents</groupId> |
|||
<artifactId>httpclient</artifactId> |
|||
<version>4.5.3</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>commons-lang</groupId> |
|||
<artifactId>commons-lang</artifactId> |
|||
<version>2.6</version> |
|||
</dependency> |
|||
<!-- https://mvnrepository.com/artifact/org.jetbrains.kotlin/kotlin-reflect --> |
|||
<dependency> |
|||
<groupId>org.jetbrains.kotlin</groupId> |
|||
<artifactId>kotlin-reflect</artifactId> |
|||
<version>1.6.21</version> |
|||
<scope>runtime</scope> |
|||
</dependency> |
|||
|
|||
<!-- Log4j 2 日志库 --> |
|||
<dependency> |
|||
<groupId>org.apache.logging.log4j</groupId> |
|||
<artifactId>log4j-api</artifactId> |
|||
<version>2.14.1</version> <!-- 或者你需要的其他版本 --> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.apache.logging.log4j</groupId> |
|||
<artifactId>log4j-core</artifactId> |
|||
<version>2.14.1</version> <!-- 或者你需要的其他版本 --> |
|||
</dependency> |
|||
|
|||
<dependency> |
|||
<groupId>org.projectlombok</groupId> |
|||
<artifactId>lombok</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>cn.hutool</groupId> |
|||
<artifactId>hutool-all</artifactId> |
|||
<version>5.8.5</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>junit</groupId> |
|||
<artifactId>junit</artifactId> |
|||
</dependency> |
|||
<!-- https://mvnrepository.com/artifact/p6spy/p6spy --> |
|||
<dependency> |
|||
<groupId>p6spy</groupId> |
|||
<artifactId>p6spy</artifactId> |
|||
<version>3.9.0</version> |
|||
</dependency> |
|||
|
|||
<dependency> |
|||
<groupId>commons-collections</groupId> |
|||
<artifactId>commons-collections</artifactId> |
|||
<version>3.2.2</version> |
|||
</dependency> |
|||
|
|||
</dependencies> |
|||
|
|||
<build> |
|||
<!-- <pluginManagement> --><!-- lock down plugins versions to avoid using Maven defaults (may be moved |
|||
to parent pom) --> |
|||
<plugins> |
|||
<!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle --> |
|||
<plugin> |
|||
<artifactId>maven-clean-plugin</artifactId> |
|||
<version>3.1.0</version> |
|||
</plugin> |
|||
<!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging --> |
|||
<plugin> |
|||
<artifactId>maven-resources-plugin</artifactId> |
|||
<version>3.0.2</version> |
|||
</plugin> |
|||
<plugin> |
|||
<artifactId>maven-compiler-plugin</artifactId> |
|||
<version>3.8.0</version> |
|||
</plugin> |
|||
<plugin> |
|||
<artifactId>maven-surefire-plugin</artifactId> |
|||
<version>2.22.1</version> |
|||
</plugin> |
|||
<plugin> |
|||
<artifactId>maven-jar-plugin</artifactId> |
|||
<version>3.0.2</version> |
|||
</plugin> |
|||
<plugin> |
|||
<artifactId>maven-install-plugin</artifactId> |
|||
<version>2.5.2</version> |
|||
</plugin> |
|||
<plugin> |
|||
<artifactId>maven-deploy-plugin</artifactId> |
|||
<version>2.8.2</version> |
|||
</plugin> |
|||
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle --> |
|||
<plugin> |
|||
<artifactId>maven-site-plugin</artifactId> |
|||
<version>3.7.1</version> |
|||
</plugin> |
|||
<plugin> |
|||
<artifactId>maven-project-info-reports-plugin</artifactId> |
|||
<version>3.0.0</version> |
|||
</plugin> |
|||
<!-- spring-boot-maven-plugin插件就是打包spring boot应用的 --> |
|||
|
|||
<plugin> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-maven-plugin</artifactId> |
|||
<configuration> |
|||
<mainClass>com.bfd.Application</mainClass> |
|||
<layout>ZIP</layout> |
|||
<includes> |
|||
<include> |
|||
<groupId>${project.groupId}</groupId> |
|||
<artifactId>${project.artifactId}</artifactId> |
|||
</include> |
|||
</includes> |
|||
</configuration> |
|||
<executions> |
|||
<execution> |
|||
<goals> |
|||
<goal>repackage</goal> |
|||
</goals> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
<plugin> |
|||
<groupId>org.apache.maven.plugins</groupId> |
|||
<artifactId>maven-dependency-plugin</artifactId> |
|||
<version>3.1.1</version> |
|||
<executions> |
|||
<execution> |
|||
<id>copy</id> |
|||
<phase>package</phase> |
|||
<goals> |
|||
<goal>copy-dependencies</goal> |
|||
</goals> |
|||
<configuration> |
|||
<type>jar</type> |
|||
<includeTypes>jar</includeTypes> |
|||
<includeScope>runtime</includeScope> |
|||
<outputDirectory>${project.build.directory}/libs</outputDirectory> |
|||
</configuration> |
|||
</execution> |
|||
</executions> |
|||
</plugin> |
|||
</plugins> |
|||
<!-- </pluginManagement> --> |
|||
</build> |
|||
</project> |
@ -0,0 +1,24 @@ |
|||
package com.bfd; |
|||
|
|||
|
|||
|
|||
|
|||
import org.springframework.boot.SpringApplication; |
|||
import org.springframework.boot.autoconfigure.SpringBootApplication; |
|||
import org.springframework.scheduling.annotation.EnableScheduling; |
|||
|
|||
/** |
|||
* 主入口 |
|||
* |
|||
* @author jian.mao |
|||
* @date 2023年7月4日 |
|||
* @description |
|||
*/ |
|||
@SpringBootApplication |
|||
@EnableScheduling |
|||
public class Application { |
|||
|
|||
public static void main(String[] args) { |
|||
SpringApplication.run(Application.class, args); |
|||
} |
|||
} |
@ -0,0 +1,37 @@ |
|||
package com.bfd.cache; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import java.util.Map; |
|||
import java.util.concurrent.LinkedBlockingDeque; |
|||
|
|||
/** |
|||
* @author jian.mao |
|||
* @date 2022年11月11日 |
|||
* @description 静态变量类 |
|||
*/ |
|||
@Slf4j |
|||
public class ConfigCache { |
|||
|
|||
/**启动条件**/ |
|||
public static boolean isStart = true; |
|||
/*****任务队列*****/ |
|||
public static LinkedBlockingDeque<Map<String, Object>> taskQueue = new LinkedBlockingDeque<Map<String,Object>>(); |
|||
/****视频生成队列***/ |
|||
public static LinkedBlockingDeque<Map<String, Object>> videoTaskQueue = new LinkedBlockingDeque<Map<String,Object>>(); |
|||
|
|||
|
|||
/** |
|||
* 队列录入任务 |
|||
* @param queue |
|||
* @param task |
|||
*/ |
|||
public static void putQueue(LinkedBlockingDeque<Map<String, Object>> queue,Map<String, Object> task){ |
|||
//next app 写入队列准备调出 |
|||
try { |
|||
queue.put(task); |
|||
} catch (InterruptedException e) { |
|||
log.error("队列写入data失败---"); |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,37 @@ |
|||
package com.bfd.controller; |
|||
|
|||
import javax.annotation.Resource; |
|||
|
|||
import org.springframework.stereotype.Controller; |
|||
import org.springframework.web.bind.annotation.PostMapping; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.ResponseBody; |
|||
|
|||
import com.bfd.service.ManagementService; |
|||
|
|||
/** |
|||
* 模型管理控制器 |
|||
* @author jian.mao |
|||
* @date 2024年7月29日 |
|||
* @description |
|||
*/ |
|||
@Controller |
|||
@RequestMapping("/model") |
|||
public class ManagementController { |
|||
@Resource |
|||
private ManagementService managementService; |
|||
|
|||
@PostMapping("/modelSelect") |
|||
@ResponseBody |
|||
public String modelSelect(@RequestBody String dataJson){ |
|||
String response = managementService.modelSelect(dataJson); |
|||
return response; |
|||
} |
|||
@PostMapping("/videoModelSelect") |
|||
@ResponseBody |
|||
public String videoModelSelect(@RequestBody String dataJson){ |
|||
String response = managementService.videoModelSelect(dataJson); |
|||
return response; |
|||
} |
|||
} |
@ -0,0 +1,183 @@ |
|||
package com.bfd.entity; |
|||
|
|||
|
|||
/** |
|||
* 常量实体类 |
|||
* @author jian.mao |
|||
* @date 2022年11月15日 |
|||
* @description |
|||
*/ |
|||
public class Constants { |
|||
|
|||
/*************************蓝图常量key名称*********************************/ |
|||
public final static String SCHEDULING = "scheduling"; |
|||
public final static String TYPE = "type"; |
|||
public final static String INTERVAL = "interval"; |
|||
public final static String CREATED = "created"; |
|||
public final static String LAST_EDIT = "last_edit"; |
|||
public final static String BLUEPRINT_ID = "blueprint_id"; |
|||
public final static String BLUEPRINTID = "blueprintId"; |
|||
public final static String BLUEPRINT_NAME = "name"; |
|||
public final static String SCENARIO = "scenario"; |
|||
public final static String AUTOCOMMITTRIGGERLAST = "autoCommitTriggerLast"; |
|||
public final static String FRESHVARIABLES = "freshVariables"; |
|||
public final static String AUTOCOMMIT = "autoCommit"; |
|||
public final static String MAXERRORS = "maxErrors"; |
|||
public final static String DATALOSS = "dataloss"; |
|||
public final static String POSITION = "position"; |
|||
public final static String SCENES_ID = "scenes_id"; |
|||
public final static String SCENESID = "scenesId"; |
|||
public final static String MULTI_BRANCH = "multi_branch"; |
|||
|
|||
public final static String SINGLE = "single"; |
|||
/**已重试次数**/ |
|||
public final static String ERROR_TIME = "error_time"; |
|||
public final static String PREVIOUS_RESULT = "previous_result"; |
|||
|
|||
/****数据id*****/ |
|||
public final static String BUSINESSKEY = "businessKey"; |
|||
|
|||
|
|||
/*************************metadata常量key名称*********************************/ |
|||
public final static String LABEL_COL = "label_col"; |
|||
public final static String LABEL = "label"; |
|||
public final static String USER = "user"; |
|||
public final static String ADMIN = "admin"; |
|||
public final static String ADDRESS = "address"; |
|||
public final static String DATASOURCE = "datasource"; |
|||
public final static String INDEX = "index"; |
|||
|
|||
/*************************app常量key名称*********************************/ |
|||
public final static String APPS = "apps"; |
|||
public final static String TRANSFER_ID = "transfer_id"; |
|||
public final static String MODULE = "module"; |
|||
public final static String VERSION = "version"; |
|||
public final static String METADATA = "metadata"; |
|||
public final static String APP_NAME = "name"; |
|||
public final static String DESCRIBE = "describe"; |
|||
public final static String NEXT_APP_ID = "next_app_id"; |
|||
public final static String EDGE_ID = "edge_id"; |
|||
public final static String START_ID = "start_id"; |
|||
public final static String END_ID = "end_id"; |
|||
|
|||
public final static String WAIT_CONDITION = "wait_condition"; |
|||
public final static String START_TAG = "start_tag"; |
|||
|
|||
/*************************module类型*********************************/ |
|||
public final static String FILE = "file"; |
|||
public final static String OCR = "OCR"; |
|||
public final static String FILTER = "Filter"; |
|||
public final static String CHATGPT = "ChatGPT"; |
|||
public final static String MYSQL = "mysql"; |
|||
|
|||
/*************************other类型*********************************/ |
|||
public final static String UNDERLINE = "_"; |
|||
public final static String RESULT_TOPIC = null; |
|||
public static final String EMPTY = ""; |
|||
public static final String HTTP = "http"; |
|||
public static final String REQUEST_ERROR_MESSAGE = "Download failed error is"; |
|||
public static final String REQUEST_RESULT = "result"; |
|||
public static final String REQUEST_RESULT_RESULTS = "results"; |
|||
public static final String MAP_TYPE = "Map"; |
|||
public static final String LIST_TYPE = "List"; |
|||
public static final String STRING_TYPE = "String"; |
|||
public static final String DOCUMENT_TYPE = "doc"; |
|||
public static final String FILTER_ZH = "过滤器"; |
|||
|
|||
public static final String JSON_SELE_SYMBOL = "$."; |
|||
public static final String LEFT_BRACKETS = "["; |
|||
public static final String RIGTH_BRACKETS = "]"; |
|||
public static final String TASKTYPE = "taskType"; |
|||
public static final Integer USER_TYPE = 1; |
|||
public static final Integer KEYWORD_TYPE = 0; |
|||
public static final Integer DETAIL_TYPE = 2; |
|||
public static final String CID = "cid"; |
|||
public static final String SITETYPE = "siteType"; |
|||
public static final Integer DEFULT_SUBJECTID = 304864; |
|||
public static final Integer DEFULT_CRAWLCYCLICITYTIME = 1440; |
|||
public static final String CRAWLENDTIME = "crawlEndTime"; |
|||
public static final String CRAWLSTARTTIME = "crawlStartTime"; |
|||
public static final String CRAWLPAGETYPES = "crawlPageTypes"; |
|||
public static final String APPID = "113ic"; |
|||
public static final String APP_ID = "appId"; |
|||
public final static String ID = "id"; |
|||
public static final Integer DEFULT_CRAWLPERIODHOUR = 24; |
|||
public static final String CREATEUSERID = "662015832180933762"; |
|||
public static final String CRAWL_ADD_URL = "https://caiji.percent.cn/api/crawl/remote/task/save"; |
|||
public static final String CRAWLKEYWORD = "crawlKeyword"; |
|||
public static final String ATTACHTAG = "attachTag"; |
|||
public static final String ATTACHTAG_VALUE = "analyze"; |
|||
public static final String KEYWORD = "keyword"; |
|||
public static final String SITEID = "siteId"; |
|||
public static final String RESULTS = "results"; |
|||
public static final String RESULT = "result"; |
|||
public static final String CRAWLDATAFLAG = "crawlDataFlag"; |
|||
public static final String CRAWLDATAFLAG_PREFIX = "\"crawlDataFlag\":\"keyword:"; |
|||
public static final String TID = "tid"; |
|||
public static final Long TIME_OUT = 1800000L; |
|||
public static final String ATTR = "attr"; |
|||
public static final String HASVIDEO = "hasVideo"; |
|||
public static final String CRAWL_END_MARK = "crawl_end_mark"; |
|||
public static final String CRAWL_END_MESSAGE = "crawl_end_message"; |
|||
public static final String CRAWL_END_MESSAGE_VALUE = "数据采集完成"; |
|||
public static final String SUBJECTID = "subjectId"; |
|||
public static final String TASKID = "taskId"; |
|||
public static final int SUCCESS_CODE = 200; |
|||
public static final String WEB_URL_SUFFIX = "/api/aogeo/api/cda/caiji/status"; |
|||
public static final String STATUS = "status"; |
|||
/************************redis*************************************/ |
|||
public static final String LOCK_KEY = "myLock"; |
|||
public static final long LOCK_EXPIRE_TIME = 300000; |
|||
|
|||
/************************应用参数*************************************/ |
|||
public static final String CODE = "code"; |
|||
public static final String MESSAGE = "message"; |
|||
public static final String INPUT = "input"; |
|||
public static final String OUTPUT = "output"; |
|||
public static final String FORM = "form"; |
|||
public static final String FIELD = "field"; |
|||
public static final String VALUE = "value"; |
|||
public static final String DATA = "data"; |
|||
public static final String COLON_EN = ":"; |
|||
public static final String DATABASE = "database"; |
|||
public static final String TABLE = "table"; |
|||
public static final String USERNAME = "username"; |
|||
public static final String PASSWORD = "password"; |
|||
public static final String PORT = "port"; |
|||
public static final String HOSTNAME = "hostname"; |
|||
public static final String DATATYPE = "dataType"; |
|||
public static final String RULES = "rules"; |
|||
public static final String GENID = "genId"; |
|||
public static final String KEY = "key"; |
|||
public static final String DATAID = "dataId"; |
|||
public static final String APP_CODE = "app_code"; |
|||
|
|||
/***************文件相关参数常量***********************/ |
|||
public static final String READCONTENT = "readContent"; |
|||
public static final String CONTENT = "content"; |
|||
public static final String JPG = "jpg"; |
|||
public static final String PNG = "png"; |
|||
public static final String JPEG = "jpeg"; |
|||
public static final String XLS = "xls"; |
|||
public static final String XLSX = "xlsx"; |
|||
public static final String PDF = "pdf"; |
|||
public static final String DOC = "doc"; |
|||
public static final String DOCX = "docx"; |
|||
public static final String PPT = "ppt"; |
|||
public static final String PPTX = "pptx"; |
|||
public static final String MP4 = "mp4"; |
|||
public static final String EXT = "ext"; |
|||
public static final String FILES = "files"; |
|||
public static final String PATH = "path"; |
|||
public static final String TITLE = "title"; |
|||
public static final String DATAPROCESSID = "dataProcessId"; |
|||
public static final String LLM = "llm"; |
|||
public static final String GPT_VALUE = "1"; |
|||
public static final String GLM_VALUE = "2"; |
|||
public static final String QWEN_VALUE = "3"; |
|||
public static final String MINIMAX_VALUE = "4"; |
|||
public static final String DOUBAO_VALUE = "5"; |
|||
public static final String MODELADDRESS = "modelAddress"; |
|||
public static final String AUTHORIZATIONS = "authorizations"; |
|||
public static final String AUTHORIZATION = "authorization"; |
|||
} |
@ -0,0 +1,132 @@ |
|||
package com.bfd.handler; |
|||
|
|||
import java.io.File; |
|||
import java.io.IOException; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.concurrent.LinkedBlockingDeque; |
|||
|
|||
import javax.annotation.Resource; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import org.apache.commons.io.FileUtils; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.ApplicationArguments; |
|||
import org.springframework.boot.ApplicationRunner; |
|||
import org.springframework.core.annotation.Order; |
|||
import org.springframework.stereotype.Component; |
|||
|
|||
import com.alibaba.fastjson.JSONObject; |
|||
import com.bfd.cache.ConfigCache; |
|||
import com.bfd.process.ModelSelectProcess; |
|||
import com.bfd.process.VideoModelSelectProcess; |
|||
import com.bfd.utils.FileUtil; |
|||
|
|||
|
|||
/** |
|||
* 启动处理入口 |
|||
* @author jian.mao |
|||
* @date 2023年11月3日 |
|||
* @description |
|||
*/ |
|||
|
|||
@Component |
|||
@Order(value = 1) |
|||
@Slf4j |
|||
public class MainHandler implements ApplicationRunner { |
|||
|
|||
@Value("${task.task-queue-path}") |
|||
private String taskPath; |
|||
@Value("${task.video-task-queue-path}") |
|||
private String videoTaskPath; |
|||
@Resource |
|||
private ModelSelectProcess modelSelectProcess; |
|||
@Resource |
|||
private VideoModelSelectProcess videoModelSelectProcess; |
|||
@Override |
|||
public void run(ApplicationArguments args) throws Exception { |
|||
new Thread(modelSelectProcess).start();; |
|||
log.info("开启模型任务分发执行线程-----"); |
|||
new Thread(videoModelSelectProcess).start();; |
|||
log.info("开启视频生成模型任务分发执行线程-----"); |
|||
//停止处理 |
|||
waitDown(); |
|||
//启动加载缓存任务 |
|||
readTask(taskPath,ConfigCache.taskQueue); |
|||
readTask(videoTaskPath,ConfigCache.videoTaskQueue); |
|||
} |
|||
|
|||
|
|||
|
|||
|
|||
|
|||
@SuppressWarnings("unchecked") |
|||
public static void readTask(String path,LinkedBlockingDeque<Map<String, Object>> queue){ |
|||
File file = new File(path); |
|||
if(file.exists()){ |
|||
List<String> tasks = null; |
|||
try { |
|||
tasks = FileUtils.readLines(file,"UTF-8"); |
|||
} catch (IOException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
for (String taskStr : tasks) { |
|||
Map<String, Object> task = JSONObject.parseObject(taskStr); |
|||
try { |
|||
queue.put(task); |
|||
} catch (InterruptedException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
file.delete(); |
|||
} |
|||
} |
|||
/** |
|||
* 结束触发钩子 |
|||
*/ |
|||
public void waitDown() { |
|||
Runtime.getRuntime().addShutdownHook(new Thread() { |
|||
@Override |
|||
public void run() { |
|||
// 停止线程 |
|||
ConfigCache.isStart = false; |
|||
log.info("stop-------"); |
|||
writeTsskToFile(); |
|||
} |
|||
}); |
|||
} |
|||
|
|||
|
|||
/** |
|||
* 任务持久化到硬盘 |
|||
*/ |
|||
public void writeTsskToFile(){ |
|||
while(true){ |
|||
if(ConfigCache.taskQueue.size() > 0 ){ |
|||
try { |
|||
Map<String, Object> task = ConfigCache.taskQueue.take(); |
|||
FileUtil.writeFile(taskPath, JSONObject.toJSONString(task)); |
|||
} catch (InterruptedException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
}else{ |
|||
log.info("taskQueue write is file end"); |
|||
break; |
|||
} |
|||
} |
|||
while(true){ |
|||
if(ConfigCache.videoTaskQueue.size() > 0 ){ |
|||
try { |
|||
Map<String, Object> task = ConfigCache.videoTaskQueue.take(); |
|||
FileUtil.writeFile(videoTaskPath, JSONObject.toJSONString(task)); |
|||
} catch (InterruptedException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
}else{ |
|||
log.info("taskQueue write is file end"); |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,74 @@ |
|||
package com.bfd.process; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Random; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import org.springframework.stereotype.Component; |
|||
|
|||
import com.alibaba.fastjson.JSONObject; |
|||
import com.bfd.cache.ConfigCache; |
|||
import com.bfd.entity.Constants; |
|||
import com.bfd.utils.DateUtil; |
|||
import com.bfd.utils.DownLoadUtil; |
|||
|
|||
|
|||
/**1 |
|||
* @author jian.mao |
|||
* @date 2024年7月29日 |
|||
* @description |
|||
*/ |
|||
@Component |
|||
@Slf4j |
|||
public class ModelSelectProcess implements Runnable{ |
|||
|
|||
private Random random = new Random(); |
|||
@Override |
|||
public void run() { |
|||
while (ConfigCache.isStart){ |
|||
try { |
|||
Map<String, Object> task = null; |
|||
task = ConfigCache.taskQueue.take(); |
|||
log.info("任务队列长度:{}", Integer.valueOf(ConfigCache.taskQueue.size())); |
|||
Map<String, Object> input = (Map<String, Object>) task.get(Constants.INPUT); |
|||
Map<String, Object> modelAddress = (Map<String, Object>) input.get(Constants.MODELADDRESS); |
|||
Map<String, Object> authorizations = (Map<String, Object>) input.get(Constants.AUTHORIZATIONS); |
|||
String llm = (String) input.get(Constants.LLM); |
|||
String address = (String) modelAddress.get(llm); |
|||
List<String> authorizationList = (List<String>) authorizations.get(llm); |
|||
String authorization = authorizationList.get(random.nextInt(authorizationList.size())); |
|||
input.put(Constants.AUTHORIZATION, authorization); |
|||
String response = null; |
|||
if(llm.equals(Constants.GPT_VALUE)){ |
|||
log.info("chatgpt 智能模型任务触发-----"); |
|||
response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task)); |
|||
}else if(llm.equals(Constants.GLM_VALUE)){ |
|||
log.info("智谱 智能模型任务触发-----"); |
|||
response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task)); |
|||
}else if(llm.equals(Constants.QWEN_VALUE)){ |
|||
log.info("本地 智能模型任务触发-----"); |
|||
response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task)); |
|||
}else if(llm.equals(Constants.MINIMAX_VALUE)){ |
|||
log.info("minimax 智能模型任务触发-----"); |
|||
response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task)); |
|||
}else if(llm.equals(Constants.DOUBAO_VALUE)){ |
|||
log.info("doubao 智能模型任务触发-----"); |
|||
response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task)); |
|||
}else{ |
|||
log.error("未知模型,不处理"); |
|||
} |
|||
if(response.contains(Constants.REQUEST_ERROR_MESSAGE)){ |
|||
log.error("接口请求异常,address:{},errormessage:{}",address,response); |
|||
}else{ |
|||
log.info("模型接口请求正常!接口地址{},接口信息:{}",address,response); |
|||
} |
|||
} catch (Throwable e) { |
|||
log.error("结果组装异常,", e); |
|||
log.info("数据流转至下游-------"); |
|||
} |
|||
|
|||
} |
|||
} |
|||
} |
@ -0,0 +1,57 @@ |
|||
package com.bfd.process; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Random; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import org.springframework.stereotype.Component; |
|||
|
|||
import com.alibaba.fastjson.JSONObject; |
|||
import com.bfd.cache.ConfigCache; |
|||
import com.bfd.entity.Constants; |
|||
import com.bfd.utils.DateUtil; |
|||
import com.bfd.utils.DownLoadUtil; |
|||
|
|||
|
|||
/**1 |
|||
* @author jian.mao |
|||
* @date 2024年7月29日 |
|||
* @description |
|||
*/ |
|||
@Component |
|||
@Slf4j |
|||
public class VideoModelSelectProcess implements Runnable{ |
|||
|
|||
private Random random = new Random(); |
|||
@Override |
|||
public void run() { |
|||
while (ConfigCache.isStart){ |
|||
try { |
|||
Map<String, Object> task = null; |
|||
task = ConfigCache.videoTaskQueue.take(); |
|||
log.info("任务队列长度:{}", Integer.valueOf(ConfigCache.videoTaskQueue.size())); |
|||
Map<String, Object> input = (Map<String, Object>) task.get(Constants.INPUT); |
|||
Map<String, Object> modelAddress = (Map<String, Object>) input.get(Constants.MODELADDRESS); |
|||
Map<String, Object> authorizations = (Map<String, Object>) input.get(Constants.AUTHORIZATIONS); |
|||
String llm = (String) input.get(Constants.LLM); |
|||
String address = (String) modelAddress.get(llm); |
|||
List<String> authorizationList = (List<String>) authorizations.get(llm); |
|||
String authorization = authorizationList.get(random.nextInt(authorizationList.size())); |
|||
input.put(Constants.AUTHORIZATION, authorization); |
|||
String response = null; |
|||
if(llm.equals(Constants.MINIMAX_VALUE)){ |
|||
log.info("minimax 视频生成模型任务触发-----"); |
|||
response = DownLoadUtil.doPost(address, JSONObject.toJSONString(task)); |
|||
}else{ |
|||
log.info("视频生成模型接口请求正常!接口地址{},接口信息:{}",address,response); |
|||
} |
|||
} catch (Throwable e) { |
|||
log.error("结果组装异常,", e); |
|||
log.info("数据流转至下游-------"); |
|||
} |
|||
|
|||
} |
|||
} |
|||
} |
@ -0,0 +1,25 @@ |
|||
package com.bfd.service; |
|||
|
|||
|
|||
/** |
|||
* |
|||
* @author jian.mao |
|||
* @date 2024年7月29日 |
|||
* @description |
|||
*/ |
|||
public interface ManagementService { |
|||
|
|||
/** |
|||
* 模型选择 |
|||
* @param dataJson |
|||
* @return |
|||
*/ |
|||
public String modelSelect(String dataJson); |
|||
|
|||
/** |
|||
* 视频模型选择 |
|||
* @param dataJson |
|||
* @return |
|||
*/ |
|||
public String videoModelSelect(String dataJson); |
|||
} |
@ -0,0 +1,77 @@ |
|||
package com.bfd.service.impl; |
|||
|
|||
import java.util.HashMap; |
|||
import java.util.Map; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import org.springframework.stereotype.Service; |
|||
|
|||
import com.alibaba.fastjson.JSONObject; |
|||
import com.bfd.cache.ConfigCache; |
|||
import com.bfd.entity.Constants; |
|||
import com.bfd.service.ManagementService; |
|||
|
|||
/** |
|||
* |
|||
* @author jian.mao |
|||
* @date 2024年7月29日 |
|||
* @description |
|||
*/ |
|||
@Service |
|||
@Slf4j |
|||
public class ManagementServiceImpl implements ManagementService { |
|||
|
|||
@Override |
|||
public String modelSelect(String dataJson) { |
|||
// TODO Auto-generated method stub |
|||
Map<String,Object> response = new HashMap<>(16); |
|||
int code = 200; |
|||
String message = "success"; |
|||
Map<String,Object> task = null; |
|||
try { |
|||
task = JSONObject.parseObject(dataJson); |
|||
} catch (Exception e) { |
|||
log.error("参数结构不合法,",e); |
|||
code = 100010; |
|||
message = "参数不合法"; |
|||
} |
|||
//写入队列 |
|||
try { |
|||
ConfigCache.taskQueue.put(task); |
|||
} catch (InterruptedException e) { |
|||
log.error("任务写入等待队列异常,",e); |
|||
code = 100011; |
|||
message = "任务写入等待队列失败"; |
|||
} |
|||
response.put(Constants.CODE,code); |
|||
response.put(Constants.MESSAGE,message); |
|||
return JSONObject.toJSONString(response); |
|||
} |
|||
@Override |
|||
public String videoModelSelect(String dataJson) { |
|||
// TODO Auto-generated method stub |
|||
Map<String,Object> response = new HashMap<>(16); |
|||
int code = 200; |
|||
String message = "success"; |
|||
Map<String,Object> task = null; |
|||
try { |
|||
task = JSONObject.parseObject(dataJson); |
|||
} catch (Exception e) { |
|||
log.error("参数结构不合法,",e); |
|||
code = 100010; |
|||
message = "参数不合法"; |
|||
} |
|||
//写入队列 |
|||
try { |
|||
ConfigCache.videoTaskQueue.put(task); |
|||
} catch (InterruptedException e) { |
|||
log.error("视频任务写入等待队列异常,",e); |
|||
code = 100011; |
|||
message = "视频任务写入等待队列失败"; |
|||
} |
|||
response.put(Constants.CODE,code); |
|||
response.put(Constants.MESSAGE,message); |
|||
return JSONObject.toJSONString(response); |
|||
} |
|||
} |
@ -0,0 +1,63 @@ |
|||
package com.bfd.utils; |
|||
|
|||
import java.util.Map; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
import com.alibaba.fastjson.JSONObject; |
|||
import com.alibaba.fastjson.JSONPath; |
|||
import com.bfd.entity.Constants; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:DataUtil |
|||
* @version:1.0 |
|||
* @description: 获取dataValue的值 |
|||
* @Date:2023/11/1 9:54 |
|||
*/ |
|||
@Slf4j |
|||
public class DataUtil { |
|||
/** |
|||
* |
|||
* @param key 传入的key |
|||
* @param dataMap 数据map |
|||
* @return 根据传入的参数进行判断解析,返回正确的dataValue |
|||
*/ |
|||
public static Object getValue(String key, Map dataMap) { |
|||
try { |
|||
//公式为空直接就返回 |
|||
if(key.equals(Constants.EMPTY)){ |
|||
return Constants.EMPTY; |
|||
} |
|||
Object dataValue; |
|||
String isJson = "#json#"; |
|||
if (key.contains(isJson)) { |
|||
//进行第一次拆分,获取#json#前面的部分 |
|||
String[] keySplit = key.split(isJson); |
|||
String firstDataKey = keySplit[0]; |
|||
String[] firstDataKeySplit = firstDataKey.split(":"); |
|||
//取出前半部分对应的JSON数据并转换为JSONObject |
|||
String dataJson = (String) dataMap.get(firstDataKeySplit[0]); |
|||
JSONObject dataJsonObject = JSON.parseObject(dataJson); |
|||
//根据key的后半部分取出对应JSONObject中的值 |
|||
String firstDataKeyJson = (String) JSONPath.eval(dataJsonObject, firstDataKeySplit[1]); |
|||
String secDataKey = keySplit[1]; |
|||
JSONObject firstDataJsonObject = JSON.parseObject(firstDataKeyJson); |
|||
dataValue = JSONPath.eval(firstDataJsonObject, secDataKey); |
|||
return dataValue; |
|||
} |
|||
String[] keySplit = key.split(":"); |
|||
String jsonPath = keySplit[1]; |
|||
String dataJson = (String) dataMap.get(keySplit[0]); |
|||
JSONObject dataJsonObject = JSON.parseObject(dataJson); |
|||
dataValue = JSONPath.eval(dataJsonObject, jsonPath); |
|||
return dataValue; |
|||
} catch (Exception e) { |
|||
// TODO: handle exception |
|||
log.error("jsonpath公式取值异常,",e); |
|||
return null; |
|||
} |
|||
|
|||
} |
|||
} |
@ -0,0 +1,177 @@ |
|||
package com.bfd.utils; |
|||
|
|||
|
|||
import java.math.BigInteger; |
|||
import java.security.MessageDigest; |
|||
import java.security.NoSuchAlgorithmException; |
|||
import java.text.ParseException; |
|||
import java.text.SimpleDateFormat; |
|||
import java.time.LocalDateTime; |
|||
import java.time.format.DateTimeFormatter; |
|||
import java.util.Date; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
import com.alibaba.fastjson.JSONObject; |
|||
|
|||
/** |
|||
* 日期工具类 |
|||
* |
|||
* @author jian.mao |
|||
* @date 2022年11月15日 |
|||
* @description |
|||
*/ |
|||
@Slf4j |
|||
public class DateUtil { |
|||
|
|||
/** |
|||
* @return |
|||
*/ |
|||
public static String getTimeStrForNow() { |
|||
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH"); |
|||
return sdf.format(new Date()); |
|||
} |
|||
|
|||
|
|||
public static String getTimeStrForDay(long time) { |
|||
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); |
|||
|
|||
return sdf.format(new Date(time * 1000)); |
|||
} |
|||
|
|||
public static String getTimeStrForDay() { |
|||
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); |
|||
|
|||
return sdf.format(new Date()); |
|||
} |
|||
|
|||
|
|||
public static String getDateTime() { |
|||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); |
|||
String time = sdf.format(new Date()); |
|||
return time; |
|||
} |
|||
|
|||
public static String getDateTime(Long timestap) { |
|||
|
|||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); |
|||
String time = sdf.format(new Date(timestap)); |
|||
return time; |
|||
} |
|||
|
|||
public static String getDate(Long timestap) { |
|||
|
|||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); |
|||
String time = sdf.format(new Date(timestap)); |
|||
return time; |
|||
} |
|||
|
|||
public static String getDateTimeForMonth() { |
|||
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM"); |
|||
String time = sdf.format(new Date()); |
|||
return time; |
|||
} |
|||
|
|||
/** |
|||
* 休眠 |
|||
* |
|||
* @param millis 毫秒 |
|||
*/ |
|||
public static void sleep(long millis) { |
|||
try { |
|||
Thread.sleep(millis); |
|||
} catch (InterruptedException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 1. @Description:时间戳转时间 |
|||
* 2. @Author: ying.zhao |
|||
* 3. @Date: 2023/3/28 |
|||
*/ |
|||
|
|||
public static String timestampToDate(String time) { |
|||
int thirteen = 13; |
|||
int ten = 10; |
|||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); |
|||
// if (time.length() == thirteen) { |
|||
if (time.length() > ten) { |
|||
return sdf.format(new Date(Long.parseLong(time))); |
|||
} else { |
|||
return sdf.format(new Date(Integer.parseInt(time) * 1000L)); |
|||
} |
|||
} |
|||
|
|||
public static String parseCreated(String jsonTime){ |
|||
String formattedDateTime = getDateTime(); |
|||
try { |
|||
// 使用fastjson解析JSON数据 |
|||
JSONObject jsonObject = JSON.parseObject(jsonTime); |
|||
// 获取日期和时间的值 |
|||
JSONObject dateObject = jsonObject.getJSONObject("date"); |
|||
int day = dateObject.getIntValue("day"); |
|||
int month = dateObject.getIntValue("month"); |
|||
int year = dateObject.getIntValue("year"); |
|||
|
|||
JSONObject timeObject = jsonObject.getJSONObject("time"); |
|||
int hour = timeObject.getIntValue("hour"); |
|||
int minute = timeObject.getIntValue("minute"); |
|||
int second = timeObject.getIntValue("second"); |
|||
|
|||
// 创建LocalDateTime对象 |
|||
LocalDateTime dateTime = LocalDateTime.of(year, month, day, hour, minute, second); |
|||
|
|||
// 定义日期时间格式化器 |
|||
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); |
|||
|
|||
// 格式化日期时间 |
|||
formattedDateTime = dateTime.format(formatter); |
|||
} catch (Exception e) { |
|||
log.info("日期转换失败:{}",e); |
|||
} |
|||
return formattedDateTime; |
|||
} |
|||
|
|||
/** |
|||
* 字符串转换日期 |
|||
* @param format |
|||
* @param date |
|||
* @return |
|||
*/ |
|||
public static Date strToDate(String format,String date){ |
|||
SimpleDateFormat sdf = new SimpleDateFormat(format); |
|||
if (date == null || date.equals("")){ |
|||
return new Date(); |
|||
}else{ |
|||
Date ru = null; |
|||
try { |
|||
ru = sdf.parse(date); |
|||
} catch (ParseException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
return ru; |
|||
} |
|||
} |
|||
/** |
|||
* 日期格式话 |
|||
* @param format 日期格式 |
|||
* @param dater 要转换的日期,默认当前时间 |
|||
* @return |
|||
*/ |
|||
public static String FormatDate(String format,Date date){ |
|||
String fromatDate = null; |
|||
SimpleDateFormat sdf = new SimpleDateFormat(format); |
|||
if (date == null){ |
|||
fromatDate = sdf.format(new Date()); |
|||
}else{ |
|||
fromatDate = sdf.format(date); |
|||
} |
|||
return fromatDate; |
|||
} |
|||
public static void main(String[] args) { |
|||
String time = timestampToDate("955814400000"); |
|||
System.out.println(time); |
|||
} |
|||
} |
1007
src/main/java/com/bfd/utils/DownLoadUtil.java
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,27 @@ |
|||
package com.bfd.utils; |
|||
|
|||
import java.security.MessageDigest; |
|||
import java.security.NoSuchAlgorithmException; |
|||
|
|||
/** |
|||
* @author jian.mao |
|||
* @date 2023年3月10日 |
|||
* @description |
|||
*/ |
|||
public class EncryptionUtil { |
|||
public static String md5(String text) { |
|||
try { |
|||
MessageDigest md = MessageDigest.getInstance("MD5"); |
|||
md.update(text.getBytes()); |
|||
byte[] bytes = md.digest(); |
|||
StringBuilder sb = new StringBuilder(); |
|||
for (byte b : bytes) { |
|||
sb.append(String.format("%02x", b & 0xff)); |
|||
} |
|||
return sb.toString(); |
|||
} catch (NoSuchAlgorithmException e) { |
|||
e.printStackTrace(); |
|||
return null; |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,36 @@ |
|||
package com.bfd.utils; |
|||
|
|||
import java.io.FileWriter; |
|||
import java.io.IOException; |
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.HashSet; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
|
|||
/** |
|||
* 文件工具类 |
|||
* @author jian.mao |
|||
* @date 2023年7月14日 |
|||
* @description |
|||
*/ |
|||
public class FileUtil { |
|||
|
|||
/** |
|||
* 数据写入文件 |
|||
* @param Path 文件路径 |
|||
* @param result 数据 |
|||
* @throws IOException |
|||
*/ |
|||
public static void writeFile(String path,String result){ |
|||
try { |
|||
FileWriter fw = new FileWriter(path,true); |
|||
fw.write(result+"\n"); |
|||
fw.flush(); |
|||
fw.close(); |
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,32 @@ |
|||
package com.bfd.utils; |
|||
|
|||
import com.alibaba.fastjson.JSONObject; |
|||
import com.bfd.entity.Constants; |
|||
|
|||
/** |
|||
* json工具 |
|||
* @author jian.mao |
|||
* @date 2023年7月10日 |
|||
* @description |
|||
*/ |
|||
public class JsonUtil { |
|||
|
|||
/** |
|||
* 校验字符串是list/map/str |
|||
* @param jsonString |
|||
* @return |
|||
*/ |
|||
public static String checkJsonType(String jsonString) { |
|||
try { |
|||
JSONObject.parseObject(jsonString); |
|||
return Constants.MAP_TYPE; |
|||
} catch (Exception e) { |
|||
try { |
|||
JSONObject.parseArray(jsonString); |
|||
return Constants.LIST_TYPE; |
|||
} catch (Exception ex) { |
|||
return Constants.STRING_TYPE; |
|||
} |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,65 @@ |
|||
package com.bfd.utils; |
|||
|
|||
import java.util.Map; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
import okhttp3.MediaType; |
|||
import okhttp3.OkHttpClient; |
|||
import okhttp3.Request; |
|||
import okhttp3.RequestBody; |
|||
import okhttp3.Response; |
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
import com.bfd.entity.Constants; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:ocrUtil |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/8/1 16:38 |
|||
*/ |
|||
public class OcrUtil { |
|||
private static OkHttpClient okHttpClient; |
|||
|
|||
private static OkHttpClient getOkHttpClient() { |
|||
if (okHttpClient == null) { |
|||
okHttpClient = new OkHttpClient(); |
|||
} |
|||
return okHttpClient; |
|||
} |
|||
|
|||
public static String doOcr(String url,String ocrApi) { |
|||
String text = ""; |
|||
int reTryTimes = 3; |
|||
for (int i = 0; i < reTryTimes; i++) { |
|||
int okCode = 200; |
|||
OkHttpClient client = getOkHttpClient(); |
|||
OkHttpClient.Builder builder = client.newBuilder().writeTimeout(600, TimeUnit.SECONDS).connectTimeout(600, TimeUnit.SECONDS).readTimeout(600, TimeUnit.SECONDS); |
|||
client = builder.build(); |
|||
MediaType mediaType = MediaType.parse("application/json"); |
|||
RequestBody body = RequestBody.create(mediaType, "{\"id\":\"\",\"url\":\"" + url + "\"}"); |
|||
Request request = new Request.Builder() |
|||
.url(ocrApi) |
|||
.method("POST", body) |
|||
.addHeader("Content-Type", "application/json") |
|||
.build(); |
|||
try { |
|||
Response response = client.newCall(request).execute(); |
|||
String html = response.body().string(); |
|||
Map dataMap = (Map) JSON.parse(html); |
|||
int code = (int) dataMap.get("code"); |
|||
if (code == okCode) { |
|||
text = (String) dataMap.get("text"); |
|||
} |
|||
if (text.equals(Constants.EMPTY)) { |
|||
break; |
|||
} |
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
|
|||
return text; |
|||
} |
|||
} |
@ -0,0 +1,33 @@ |
|||
package com.bfd.utils; |
|||
|
|||
import java.security.MessageDigest; |
|||
|
|||
/** |
|||
* 其他工具类 |
|||
* @author jian.mao |
|||
* @date 2023年9月19日 |
|||
* @description |
|||
*/ |
|||
public class OtherUtils { |
|||
|
|||
|
|||
|
|||
public static String getMd5(String string) { |
|||
try { |
|||
MessageDigest md5 = MessageDigest.getInstance("MD5"); |
|||
byte[] bs = md5.digest(string.getBytes("UTF-8")); |
|||
StringBuilder sb = new StringBuilder(40); |
|||
for (byte x : bs) { |
|||
if ((x & 0xff) >> 4 == 0) { |
|||
sb.append("0").append(Integer.toHexString(x & 0xff)); |
|||
} else { |
|||
sb.append(Integer.toHexString(x & 0xff)); |
|||
} |
|||
} |
|||
return sb.toString(); |
|||
} catch (Exception e) { |
|||
|
|||
return "nceaform" + System.currentTimeMillis(); |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,18 @@ |
|||
package com.bfd.utils; |
|||
|
|||
import java.util.Map; |
|||
import java.util.concurrent.LinkedBlockingDeque; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:QueueUtil |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/7/13 15:00 |
|||
*/ |
|||
public class QueueUtil { |
|||
|
|||
public static LinkedBlockingDeque<Map<String, Object>> taskQueue = new LinkedBlockingDeque<Map<String, Object>>(); |
|||
|
|||
public static LinkedBlockingDeque<String> sendQueue = new LinkedBlockingDeque<String>(); |
|||
} |
@ -0,0 +1,23 @@ |
|||
package com.bfd.utils; |
|||
|
|||
import java.io.PrintWriter; |
|||
import java.io.StringWriter; |
|||
|
|||
/** |
|||
* @author jian.mao |
|||
* @date 2023年3月22日 |
|||
* @description |
|||
*/ |
|||
public class ThrowMessageUtil { |
|||
|
|||
/** |
|||
* 获取异常信息 |
|||
* @param t |
|||
* @return |
|||
*/ |
|||
public static String getErrmessage(Throwable t){ |
|||
StringWriter stringWriter=new StringWriter(); |
|||
t.printStackTrace(new PrintWriter(stringWriter,true)); |
|||
return stringWriter.getBuffer().toString(); |
|||
} |
|||
} |
@ -0,0 +1,92 @@ |
|||
logging: |
|||
level: |
|||
root: info |
|||
path: ../logs |
|||
server: |
|||
port: 8018 |
|||
servlet: |
|||
context-path: /IntelliModelManager |
|||
tomcat: |
|||
uri-encoding: utf-8 |
|||
max-connections: 20000 |
|||
max-http-form-post-size: 1 |
|||
max-threads: 1000 |
|||
spring: |
|||
application: |
|||
name: 智能模型管理 |
|||
datasource: |
|||
url: jdbc:mysql://172.18.1.147:3306/youzhi_cda_db?serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8&useSSL=true |
|||
username: root |
|||
password: baifendian123 |
|||
driver-class-name: com.mysql.cj.jdbc.Driver |
|||
kafka: |
|||
bootstrap-servers: 172.18.1.146:9092,172.18.1.147:9092,172.18.1.148:9092 |
|||
producer: |
|||
retries: 0 |
|||
#当有多个消息需要被发送到同一个分区时,生产者会把它们放在同一个批次里。该参数指定了一个批次可以使用的内存大小,按照字节数计算。 |
|||
batch-size: 16384 |
|||
# 设置生产者内存缓冲区的大小。 |
|||
buffer-memory: 33554432 |
|||
# 键的序列化方式 |
|||
key-serializer: org.apache.kafka.common.serialization.StringSerializer |
|||
# 值的序列化方式 |
|||
value-serializer: org.apache.kafka.common.serialization.StringSerializer |
|||
# acks=0 : 生产者在成功写入消息之前不会等待任何来自服务器的响应。 |
|||
# acks=1 : 只要集群的首领节点收到消息,生产者就会收到一个来自服务器成功响应。 |
|||
# acks=all :只有当所有参与复制的节点全部收到消息时,生产者才会收到一个来自服务器的成功响应。 |
|||
acks: 1 |
|||
consumer: |
|||
# 自动提交的时间间隔 在spring boot 2.X 版本中这里采用的是值的类型为Duration 需要符合特定的格式,如1S,1M,2H,5D |
|||
auto-commit-interval: 1S |
|||
# 该属性指定了消费者在读取一个没有偏移量的分区或者偏移量无效的情况下该作何处理: |
|||
# latest(默认值)在偏移量无效的情况下,消费者将从最新的记录开始读取数据(在消费者启动之后生成的记录) |
|||
# earliest :在偏移量无效的情况下,消费者将从起始位置读取分区的记录 |
|||
auto-offset-reset: earliest |
|||
# 是否自动提交偏移量,默认值是true,为了避免出现重复数据和数据丢失,可以把它设置为false,然后手动提交偏移量 |
|||
enable-auto-commit: true |
|||
# 键的反序列化方式 |
|||
key-deserializer: org.apache.kafka.common.serialization.StringDeserializer |
|||
# 值的反序列化方式 |
|||
value-deserializer: org.apache.kafka.common.serialization.StringDeserializer |
|||
#消费组 |
|||
group-id: test4 |
|||
#消费者并发线程数 |
|||
concurrency: 4 |
|||
#超时时间 |
|||
max-poll-interval-ms: 60000 |
|||
#listener: |
|||
# 在侦听器容器中运行的线程数。 |
|||
#concurrency: 5 |
|||
#listner负责ack,每调用一次,就立即commit |
|||
#ack-mode: manual_immediate |
|||
#missing-topics-fatal: false |
|||
boot: |
|||
admin: |
|||
client: |
|||
url: http://172.18.1.147:8001 |
|||
instance: |
|||
service-base-url: http://172.18.1.147:8018 |
|||
|
|||
management: |
|||
endpoints: |
|||
web: |
|||
exposure: |
|||
include: "*" |
|||
endpoint: |
|||
health: |
|||
show-details: always |
|||
health: |
|||
elasticsearch: |
|||
enabled: false |
|||
|
|||
customize-kafka: |
|||
producer: |
|||
topic: produce_analyze |
|||
task: |
|||
task-queue-path: ../data/taskQueue.txt |
|||
video-task-queue-path: ../data/videoTaskQueue.txt |
|||
thread-num: 1 |
|||
file: |
|||
path: ../files/ |
|||
gofast: |
|||
url: http://172.18.1.180:9980/group17/upload |
@ -0,0 +1,36 @@ |
|||
<configuration> |
|||
<!-- 属性文件:在properties文件中找到对应的配置项 --> |
|||
<springProperty scope="context" name="logging.path" source="logging.path"/> |
|||
<springProperty scope="context" name="logging.level" source="logging.level.com.bfd"/> |
|||
<!-- 默认的控制台日志输出,一般生产环境都是后台启动,这个没太大作用 --> |
|||
<!-- <appender name="STDOUT" |
|||
class="ch.qos.logback.core.ConsoleAppender"> |
|||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> |
|||
<Pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</Pattern> |
|||
</encoder> |
|||
</appender> --> |
|||
|
|||
<appender name="GLMAPPER-LOGGERONE" |
|||
class="ch.qos.logback.core.rolling.RollingFileAppender"> |
|||
<append>true</append> |
|||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
|||
<level>${logging.level}</level> |
|||
</filter> |
|||
<file> |
|||
${logging.path}/IntelliModelManagerInfo.log |
|||
</file> |
|||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> |
|||
<FileNamePattern>${logging.path}/IntelliModelManagerInfo.log.%d{yyyy-MM-dd}</FileNamePattern> |
|||
<MaxHistory>7</MaxHistory> |
|||
</rollingPolicy> |
|||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> |
|||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</pattern> |
|||
<charset>UTF-8</charset> |
|||
</encoder> |
|||
</appender> |
|||
|
|||
<root level="info"> |
|||
<appender-ref ref="GLMAPPER-LOGGERONE"/> |
|||
<!-- <appender-ref ref="STDOUT"/> --> |
|||
</root> |
|||
</configuration> |
@ -0,0 +1,20 @@ |
|||
package com.bfd; |
|||
|
|||
import static org.junit.Assert.assertTrue; |
|||
|
|||
import org.junit.Test; |
|||
|
|||
/** |
|||
* Unit test for simple App. |
|||
*/ |
|||
public class AppTest |
|||
{ |
|||
/** |
|||
* Rigorous Test :-) |
|||
*/ |
|||
@Test |
|||
public void shouldAnswerWithTrue() |
|||
{ |
|||
assertTrue( true ); |
|||
} |
|||
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue