commit
cb8c11dfdd
24 changed files with 1483 additions and 0 deletions
-
33.gitignore
-
1README.md
-
138pom.xml
-
13src/main/java/com/bfd/crawl/ppthandler/PptHanlerApplication.java
-
60src/main/java/com/bfd/crawl/ppthandler/bean/ResponsePo.java
-
48src/main/java/com/bfd/crawl/ppthandler/config/AsyncThreadConfiguration.java
-
27src/main/java/com/bfd/crawl/ppthandler/config/Constant.java
-
46src/main/java/com/bfd/crawl/ppthandler/controller/PptHandlerController.java
-
32src/main/java/com/bfd/crawl/ppthandler/enums/ResponseCode.java
-
121src/main/java/com/bfd/crawl/ppthandler/service/HandlerService.java
-
54src/main/java/com/bfd/crawl/ppthandler/service/SendService.java
-
63src/main/java/com/bfd/crawl/ppthandler/service/StartServcie.java
-
44src/main/java/com/bfd/crawl/ppthandler/util/CsvUtil.java
-
60src/main/java/com/bfd/crawl/ppthandler/util/DataUtil.java
-
182src/main/java/com/bfd/crawl/ppthandler/util/ExcelUtils.java
-
117src/main/java/com/bfd/crawl/ppthandler/util/FileDownloader.java
-
42src/main/java/com/bfd/crawl/ppthandler/util/FileUtil.java
-
59src/main/java/com/bfd/crawl/ppthandler/util/OcrUtil.java
-
23src/main/java/com/bfd/crawl/ppthandler/util/OsUtil.java
-
124src/main/java/com/bfd/crawl/ppthandler/util/PptUtil.java
-
19src/main/java/com/bfd/crawl/ppthandler/util/QueueUtil.java
-
94src/main/java/com/bfd/crawl/ppthandler/util/StringUtil.java
-
47src/main/resources/application.yml
-
36src/main/resources/logback-spring.xml
@ -0,0 +1,33 @@ |
|||||
|
HELP.md |
||||
|
target/ |
||||
|
!.mvn/wrapper/maven-wrapper.jar |
||||
|
!**/src/main/**/target/ |
||||
|
!**/src/test/**/target/ |
||||
|
|
||||
|
### STS ### |
||||
|
.apt_generated |
||||
|
.classpath |
||||
|
.factorypath |
||||
|
.project |
||||
|
.settings |
||||
|
.springBeans |
||||
|
.sts4-cache |
||||
|
|
||||
|
### IntelliJ IDEA ### |
||||
|
.idea |
||||
|
*.iws |
||||
|
*.iml |
||||
|
*.ipr |
||||
|
|
||||
|
### NetBeans ### |
||||
|
/nbproject/private/ |
||||
|
/nbbuild/ |
||||
|
/dist/ |
||||
|
/nbdist/ |
||||
|
/.nb-gradle/ |
||||
|
build/ |
||||
|
!**/src/main/**/build/ |
||||
|
!**/src/test/**/build/ |
||||
|
|
||||
|
### VS Code ### |
||||
|
.vscode/ |
@ -0,0 +1 @@ |
|||||
|
ppt 解析应用 |
@ -0,0 +1,138 @@ |
|||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
<parent> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter-parent</artifactId> |
||||
|
<version>2.2.4.RELEASE</version> |
||||
|
<relativePath/> <!-- lookup parent from repository --> |
||||
|
</parent> |
||||
|
<groupId>com.bfd.crawl</groupId> |
||||
|
<artifactId>pptHandler</artifactId> |
||||
|
<version>0.0.1-SNAPSHOT</version> |
||||
|
<name>pptHandler</name> |
||||
|
<description>pptHandler</description> |
||||
|
<properties> |
||||
|
<java.version>8</java.version> |
||||
|
</properties> |
||||
|
|
||||
|
|
||||
|
<dependencies> |
||||
|
<!-- https://mvnrepository.com/artifact/de.codecentric/spring-boot-admin-starter-client --> |
||||
|
<dependency> |
||||
|
<groupId>de.codecentric</groupId> |
||||
|
<artifactId>spring-boot-admin-client</artifactId> |
||||
|
<version>2.2.4</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.apache.commons</groupId> |
||||
|
<artifactId>commons-csv</artifactId> |
||||
|
<version>1.10.0</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.kafka</groupId> |
||||
|
<artifactId>spring-kafka</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter-web</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-devtools</artifactId> |
||||
|
<scope>runtime</scope> |
||||
|
<optional>true</optional> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.projectlombok</groupId> |
||||
|
<artifactId>lombok</artifactId> |
||||
|
<optional>true</optional> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter-test</artifactId> |
||||
|
<scope>test</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.apache.pdfbox</groupId> |
||||
|
<artifactId>pdfbox</artifactId> |
||||
|
<version>2.0.28</version> |
||||
|
</dependency> |
||||
|
<!-- Apache POI --> |
||||
|
<dependency> |
||||
|
<groupId>org.apache.poi</groupId> |
||||
|
<artifactId>poi-scratchpad</artifactId> |
||||
|
<version>5.2.0</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.apache.poi</groupId> |
||||
|
<artifactId>poi</artifactId> |
||||
|
<version>5.2.0</version> |
||||
|
<exclusions> |
||||
|
<exclusion> |
||||
|
<groupId>org.apache.logging.log4j</groupId> |
||||
|
<artifactId>log4j-api</artifactId> |
||||
|
</exclusion> |
||||
|
</exclusions> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.apache.logging.log4j</groupId> |
||||
|
<artifactId>log4j-api</artifactId> |
||||
|
<version>2.17.1</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.apache.poi</groupId> |
||||
|
<artifactId>poi-ooxml</artifactId> |
||||
|
<version>5.2.0</version> |
||||
|
</dependency> |
||||
|
<!--JSON--> |
||||
|
<dependency> |
||||
|
<groupId>com.alibaba</groupId> |
||||
|
<artifactId>fastjson</artifactId> |
||||
|
<version>2.0.17</version> |
||||
|
</dependency> |
||||
|
<!--OKHTTP--> |
||||
|
<dependency> |
||||
|
<groupId>com.squareup.okhttp3</groupId> |
||||
|
<artifactId>okhttp</artifactId> |
||||
|
<version>3.9.1</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>com.google.code.gson</groupId> |
||||
|
<artifactId>gson</artifactId> |
||||
|
<version>2.8.8</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.apache.kafka</groupId> |
||||
|
<artifactId>kafka-clients</artifactId> |
||||
|
<version>2.3.1</version> <!--根据您正在使用的Kafka版本选择合适的版本号--> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.kafka</groupId> |
||||
|
<artifactId>spring-kafka-test</artifactId> |
||||
|
<scope>test</scope> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
<build> |
||||
|
<plugins> |
||||
|
<plugin> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-maven-plugin</artifactId> |
||||
|
<configuration> |
||||
|
<excludes> |
||||
|
<exclude> |
||||
|
<groupId>org.projectlombok</groupId> |
||||
|
<artifactId>lombok</artifactId> |
||||
|
</exclude> |
||||
|
</excludes> |
||||
|
</configuration> |
||||
|
</plugin> |
||||
|
</plugins> |
||||
|
</build> |
||||
|
|
||||
|
</project> |
@ -0,0 +1,13 @@ |
|||||
|
package com.bfd.crawl.ppthandler; |
||||
|
|
||||
|
import org.springframework.boot.SpringApplication; |
||||
|
import org.springframework.boot.autoconfigure.SpringBootApplication; |
||||
|
|
||||
|
@SpringBootApplication |
||||
|
public class PptHanlerApplication { |
||||
|
|
||||
|
public static void main(String[] args) { |
||||
|
SpringApplication.run(PptHanlerApplication.class, args); |
||||
|
} |
||||
|
|
||||
|
} |
@ -0,0 +1,60 @@ |
|||||
|
package com.bfd.crawl.ppthandler.bean; |
||||
|
|
||||
|
|
||||
|
|
||||
|
import com.bfd.crawl.ppthandler.enums.ResponseCode; |
||||
|
import lombok.AllArgsConstructor; |
||||
|
import lombok.Data; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:ResponsePo |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/4/3 17:23 |
||||
|
*/ |
||||
|
@Data |
||||
|
@NoArgsConstructor |
||||
|
@AllArgsConstructor |
||||
|
public class ResponsePo { |
||||
|
/** |
||||
|
* 响应码 |
||||
|
*/ |
||||
|
private int code; |
||||
|
|
||||
|
/** |
||||
|
* 正常放 返回数据 的JSON串 |
||||
|
*/ |
||||
|
private Object data; |
||||
|
|
||||
|
/** |
||||
|
* 提示消息 |
||||
|
*/ |
||||
|
private String message; |
||||
|
|
||||
|
public static ResponsePo success() { |
||||
|
return setStatus(ResponseCode.SUCCESS.getCode(), ResponseCode.SUCCESS.getMessage()); |
||||
|
} |
||||
|
|
||||
|
public static ResponsePo error() { |
||||
|
return setStatus(ResponseCode.FAILURE.getCode(), ResponseCode.FAILURE.getMessage()); |
||||
|
} |
||||
|
|
||||
|
public static ResponsePo setStatus(int code, String message) { |
||||
|
ResponsePo resultBean = new ResponsePo(); |
||||
|
resultBean.code = code; |
||||
|
resultBean.message = message; |
||||
|
return resultBean; |
||||
|
} |
||||
|
public ResponsePo(int code, String message) { |
||||
|
this.code = code; |
||||
|
this.message = message; |
||||
|
this.data = data; |
||||
|
} |
||||
|
public ResponsePo(ResponseCode responseCode){ |
||||
|
this.code = responseCode.getCode(); |
||||
|
this.message = responseCode.getMessage(); |
||||
|
this.data = data; |
||||
|
} |
||||
|
} |
@ -0,0 +1,48 @@ |
|||||
|
package com.bfd.crawl.ppthandler.config; |
||||
|
|
||||
|
|
||||
|
import org.springframework.context.annotation.Bean; |
||||
|
import org.springframework.context.annotation.Configuration; |
||||
|
import org.springframework.scheduling.annotation.EnableAsync; |
||||
|
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; |
||||
|
|
||||
|
import java.util.concurrent.Executor; |
||||
|
|
||||
|
/** |
||||
|
* @author jinming |
||||
|
* @version 1.0 |
||||
|
* @className AsyncThreadConfiguration |
||||
|
* @Date 2022/2/17 18:37 |
||||
|
*/ |
||||
|
@Configuration |
||||
|
@EnableAsync |
||||
|
public class AsyncThreadConfiguration { |
||||
|
@Bean |
||||
|
public Executor asyncExecutor() { |
||||
|
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); |
||||
|
// 核心线程数 |
||||
|
executor.setCorePoolSize(500); |
||||
|
// 并发线程的数量限制为2 |
||||
|
executor.setMaxPoolSize(500); |
||||
|
// 线程队列 |
||||
|
executor.setQueueCapacity(500); |
||||
|
executor.setThreadNamePrefix("handlerData-"); |
||||
|
executor.initialize(); |
||||
|
executor.setWaitForTasksToCompleteOnShutdown(true); |
||||
|
return executor; |
||||
|
} |
||||
|
@Bean |
||||
|
public Executor sendExecutor() { |
||||
|
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); |
||||
|
// 核心线程数 |
||||
|
executor.setCorePoolSize(500); |
||||
|
// 并发线程的数量限制为2 |
||||
|
executor.setMaxPoolSize(500); |
||||
|
// 线程队列 |
||||
|
executor.setQueueCapacity(500); |
||||
|
executor.setThreadNamePrefix("sendData-"); |
||||
|
executor.initialize(); |
||||
|
executor.setWaitForTasksToCompleteOnShutdown(true); |
||||
|
return executor; |
||||
|
} |
||||
|
} |
@ -0,0 +1,27 @@ |
|||||
|
package com.bfd.crawl.ppthandler.config; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:Constant |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/8/16 15:26 |
||||
|
*/ |
||||
|
public class Constant { |
||||
|
/** |
||||
|
* |
||||
|
*/ |
||||
|
public final static String IS_XLS = "xls"; |
||||
|
|
||||
|
/** |
||||
|
* |
||||
|
*/ |
||||
|
public final static String IS_CSV = "csv"; |
||||
|
|
||||
|
/** |
||||
|
* |
||||
|
*/ |
||||
|
public final static String ALL = "*"; |
||||
|
|
||||
|
|
||||
|
} |
@ -0,0 +1,46 @@ |
|||||
|
package com.bfd.crawl.ppthandler.controller; |
||||
|
|
||||
|
|
||||
|
import com.alibaba.fastjson.JSON; |
||||
|
|
||||
|
import com.bfd.crawl.ppthandler.bean.ResponsePo; |
||||
|
import com.bfd.crawl.ppthandler.enums.ResponseCode; |
||||
|
import com.bfd.crawl.ppthandler.util.QueueUtil; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.web.bind.annotation.PostMapping; |
||||
|
import org.springframework.web.bind.annotation.RequestBody; |
||||
|
import org.springframework.web.bind.annotation.RequestMapping; |
||||
|
import org.springframework.web.bind.annotation.RestController; |
||||
|
|
||||
|
import java.util.Map; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:DataFilterController |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/7/26 11:21 |
||||
|
*/ |
||||
|
@RestController |
||||
|
@RequestMapping("/handlerdata") |
||||
|
@Slf4j |
||||
|
public class PptHandlerController { |
||||
|
@PostMapping("/ppthandler") |
||||
|
public ResponsePo documentFeedback(@RequestBody String dataJson) { |
||||
|
|
||||
|
ResponsePo responsePo = ResponsePo.success(); |
||||
|
try { |
||||
|
Map parse = (Map) JSON.parse(dataJson); |
||||
|
} catch (Exception e) { |
||||
|
e.printStackTrace(); |
||||
|
log.error("请求格式发生异常" + e.getMessage()); |
||||
|
responsePo.setCode(ResponseCode.FAILURE.getCode()); |
||||
|
responsePo.setMessage(ResponseCode.FAILURE.getMessage()); |
||||
|
return responsePo; |
||||
|
} |
||||
|
log.info("新增任务:" + dataJson); |
||||
|
QueueUtil.taskQueue.add(dataJson); |
||||
|
|
||||
|
return responsePo; |
||||
|
} |
||||
|
} |
@ -0,0 +1,32 @@ |
|||||
|
package com.bfd.crawl.ppthandler.enums; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:ResponseCodeEnum |
||||
|
* @version:1.0 |
||||
|
* @description:响应结果码枚举类 |
||||
|
* @Date:2023/2/28 11:40 |
||||
|
*/ |
||||
|
public enum ResponseCode { |
||||
|
//返回结果码枚举类 |
||||
|
SUCCESS(200, "操作成功"), |
||||
|
FAILURE(400, "参数错误"), |
||||
|
INTERNAL_SERVER_ERROR(500, "服务器内部错误"), |
||||
|
TYPE_NOT_SUPPORT(601,"文件类型不支持"); |
||||
|
|
||||
|
private int code; |
||||
|
private String message; |
||||
|
|
||||
|
ResponseCode(int code, String message) { |
||||
|
this.code = code; |
||||
|
this.message = message; |
||||
|
} |
||||
|
|
||||
|
public int getCode() { |
||||
|
return code; |
||||
|
} |
||||
|
|
||||
|
public String getMessage() { |
||||
|
return message; |
||||
|
} |
||||
|
} |
@ -0,0 +1,121 @@ |
|||||
|
package com.bfd.crawl.ppthandler.service; |
||||
|
|
||||
|
|
||||
|
import com.alibaba.fastjson.JSON; |
||||
|
import com.bfd.crawl.ppthandler.config.Constant; |
||||
|
import com.bfd.crawl.ppthandler.util.*; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.scheduling.annotation.Async; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.io.IOException; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.Set; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:HandlerService |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/8/1 16:05 |
||||
|
*/ |
||||
|
@Service |
||||
|
@Slf4j |
||||
|
public class HandlerService { |
||||
|
@Value("${file.path}") |
||||
|
private String downloadFilePath; |
||||
|
|
||||
|
@Async("asyncExecutor") |
||||
|
void run() { |
||||
|
while (true) { |
||||
|
try { |
||||
|
if (QueueUtil.taskQueue.size() > 0) { |
||||
|
String dataJson = null; |
||||
|
try { |
||||
|
dataJson = QueueUtil.taskQueue.take(); |
||||
|
} catch (InterruptedException e) { |
||||
|
e.printStackTrace(); |
||||
|
continue; |
||||
|
} |
||||
|
String errorMessage = ""; |
||||
|
Map parse = (Map) JSON.parse(dataJson); |
||||
|
Map dataMap = (Map) parse.get("data"); |
||||
|
int id = (int) parse.get("id"); |
||||
|
Map admin = (Map) parse.get("input"); |
||||
|
|
||||
|
String key = (String) admin.get("fileUrl"); |
||||
|
String fileUrl = (String) DataUtil.getValue(key, dataMap); |
||||
|
boolean windows = OsUtil.isWindows(); |
||||
|
Map output = (Map) parse.get("output"); |
||||
|
// List<String> sheetName = (List<String>) admin.get("sheetName"); |
||||
|
//fieldType:自定义输出字段: 0 关闭,1-开启,如果开启则拼接form到output里(如果关闭,则取默认的output拼接) |
||||
|
String fileType = StringUtil.getStrByPattern(fileUrl, "\\.[^.\\\\/:*?\"<>|\\r\\n]+$"); |
||||
|
log.info("任务:" + id + "的文件类型为" + fileType); |
||||
|
String fileName = StringUtil.getMd5(fileUrl); |
||||
|
//定义ppt的下载路径 |
||||
|
String pptDir = null; |
||||
|
try { |
||||
|
pptDir = downloadFilePath.concat(windows ? "\\ppt\\" : "/ppt/").concat(fileName).concat(fileType); |
||||
|
FileDownloader.downloadFile(fileUrl, new File(pptDir)); |
||||
|
} catch (IOException e) { |
||||
|
e.printStackTrace(); |
||||
|
errorMessage = "文件下载失败"; |
||||
|
} |
||||
|
String pptParse = null; |
||||
|
String pptx = "pptx"; |
||||
|
try { |
||||
|
if (fileType.contains(pptx)) { |
||||
|
pptParse = PptUtil.parsepptx(pptDir); |
||||
|
} else { |
||||
|
pptParse = PptUtil.parsePpt(pptDir); |
||||
|
} |
||||
|
|
||||
|
} catch (Exception e) { |
||||
|
e.printStackTrace(); |
||||
|
errorMessage = "文件解析失败"; |
||||
|
} |
||||
|
|
||||
|
if (!StringUtil.hasValue(errorMessage)) { |
||||
|
Map result = new HashMap(32); |
||||
|
Map resultsMap = new HashMap(32); |
||||
|
resultsMap.put("result", pptParse); |
||||
|
resultsMap.put("isLast", 1); |
||||
|
result.put("status", 1); |
||||
|
result.put("message", "成功"); |
||||
|
result.put("results", JSON.toJSONString(resultsMap)); |
||||
|
parse.put("result", result); |
||||
|
String message = JSON.toJSONString(parse); |
||||
|
QueueUtil.sendQueue.put(message); |
||||
|
continue; |
||||
|
} |
||||
|
Map result = new HashMap(32); |
||||
|
Map resultsMap = new HashMap(32); |
||||
|
resultsMap.put("content", errorMessage); |
||||
|
resultsMap.put("isLast", 1); |
||||
|
result.put("results", JSON.toJSONString(resultsMap)); |
||||
|
result.put("status", 2); |
||||
|
result.put("message", errorMessage); |
||||
|
parse.put("result", result); |
||||
|
String message = JSON.toJSONString(parse); |
||||
|
QueueUtil.sendQueue.put(message); |
||||
|
} else { |
||||
|
log.info("任务队列为空,休眠3秒"); |
||||
|
try { |
||||
|
Thread.sleep(3000); |
||||
|
} catch (InterruptedException e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
} |
||||
|
} catch (Throwable e) { |
||||
|
e.printStackTrace(); |
||||
|
log.error("工作线程发生异常" + e.getMessage()); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
} |
@ -0,0 +1,54 @@ |
|||||
|
package com.bfd.crawl.ppthandler.service; |
||||
|
|
||||
|
|
||||
|
import com.alibaba.fastjson.JSON; |
||||
|
import com.bfd.crawl.ppthandler.util.QueueUtil; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.kafka.core.KafkaTemplate; |
||||
|
import org.springframework.scheduling.annotation.Async; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
|
||||
|
import javax.annotation.Resource; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:SendService |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/7/31 17:53 |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
@Service |
||||
|
public class SendService { |
||||
|
@Value("${send.topic}") |
||||
|
private String topic; |
||||
|
|
||||
|
@Resource |
||||
|
private KafkaTemplate kafkaTemplate; |
||||
|
|
||||
|
@Async("sendExecutor") |
||||
|
void sendToKafka() { |
||||
|
while (true) { |
||||
|
if (QueueUtil.sendQueue.size() > 0) { |
||||
|
try { |
||||
|
String message = QueueUtil.sendQueue.take(); |
||||
|
Map parse = (Map) JSON.parse(message); |
||||
|
String id = parse.get("id").toString(); |
||||
|
log.info("ID:" + id + "\t" + "数据已发出"); |
||||
|
kafkaTemplate.send(topic, message); |
||||
|
} catch (Exception e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
} else { |
||||
|
log.info("任务队列为空,休眠3秒"); |
||||
|
try { |
||||
|
Thread.sleep(3000); |
||||
|
} catch (InterruptedException e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
@ -0,0 +1,63 @@ |
|||||
|
package com.bfd.crawl.ppthandler.service; |
||||
|
|
||||
|
|
||||
|
import com.bfd.crawl.ppthandler.util.QueueUtil; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.boot.ApplicationArguments; |
||||
|
import org.springframework.boot.ApplicationRunner; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:StartServcie |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/7/31 17:14 |
||||
|
*/ |
||||
|
@Service |
||||
|
@Slf4j |
||||
|
public class StartServcie implements ApplicationRunner { |
||||
|
@Value("${thread.handler}") |
||||
|
private int handlerNumber; |
||||
|
@Value("${thread.send}") |
||||
|
private int sendNumber; |
||||
|
|
||||
|
@Autowired |
||||
|
private HandlerService handlerService; |
||||
|
@Autowired |
||||
|
private SendService sendService; |
||||
|
|
||||
|
@Override |
||||
|
public void run(ApplicationArguments args) throws Exception { |
||||
|
for (int i = 0; i < handlerNumber; i++) { |
||||
|
log.info("处理服务线程" + i + "已启动 "); |
||||
|
handlerService.run(); |
||||
|
} |
||||
|
for (int i = 0; i < sendNumber; i++) { |
||||
|
log.info("发送服务线程" + i + "已启动 "); |
||||
|
sendService.sendToKafka(); |
||||
|
} |
||||
|
Runnable myRunnable = new Runnable() { |
||||
|
@Override |
||||
|
public void run() { |
||||
|
while (true) { |
||||
|
log.info("任务队列长度为" + QueueUtil.taskQueue.size()); |
||||
|
log.info("发送队列长度为" + QueueUtil.sendQueue.size()); |
||||
|
try { |
||||
|
Thread.sleep(10000); |
||||
|
} catch (InterruptedException e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
}; |
||||
|
// 创建一个新的线程,并将Runnable对象传递给Thread构造函数 |
||||
|
Thread myThread = new Thread(myRunnable); |
||||
|
// 启动线程 |
||||
|
myThread.start(); |
||||
|
|
||||
|
|
||||
|
} |
||||
|
} |
@ -0,0 +1,44 @@ |
|||||
|
package com.bfd.crawl.ppthandler.util; |
||||
|
|
||||
|
import org.apache.commons.csv.CSVFormat; |
||||
|
import org.apache.commons.csv.CSVParser; |
||||
|
import org.apache.commons.csv.CSVRecord; |
||||
|
|
||||
|
import java.io.FileInputStream; |
||||
|
import java.io.InputStreamReader; |
||||
|
import java.io.Reader; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:CsvUtil |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2024/3/25 11:02 |
||||
|
*/ |
||||
|
public class CsvUtil { |
||||
|
public static List<Map<String, String>> parseCSV(String filePath, String csvCharSet) { |
||||
|
List<Map<String, String>> dataList = new ArrayList<>(); |
||||
|
try (Reader reader = new InputStreamReader(new FileInputStream(filePath), csvCharSet); |
||||
|
CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT.withFirstRecordAsHeader())) { |
||||
|
Map<String, Integer> headerMap = csvParser.getHeaderMap(); |
||||
|
for (CSVRecord csvRecord : csvParser) { |
||||
|
Map<String, String> dataMap = new HashMap<>(); |
||||
|
for (Map.Entry<String, Integer> entry : headerMap.entrySet()) { |
||||
|
String key = entry.getKey(); |
||||
|
int index = entry.getValue(); |
||||
|
dataMap.put(key, csvRecord.get(index)); |
||||
|
} |
||||
|
dataList.add(dataMap); |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
|
||||
|
return dataList; |
||||
|
} |
||||
|
|
||||
|
} |
@ -0,0 +1,60 @@ |
|||||
|
package com.bfd.crawl.ppthandler.util; |
||||
|
|
||||
|
import com.alibaba.fastjson.JSON; |
||||
|
import com.alibaba.fastjson.JSONObject; |
||||
|
import com.alibaba.fastjson.JSONPath; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
|
||||
|
import java.util.Map; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:DataUtil |
||||
|
* @version:1.0 |
||||
|
* @description: 获取dataValue的值 |
||||
|
* @Date:2023/11/1 9:54 |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public class DataUtil { |
||||
|
/** |
||||
|
* @param key 传入的key |
||||
|
* @param dataMap 数据map |
||||
|
* @return 根据传入的参数进行判断解析,返回正确的dataValue |
||||
|
*/ |
||||
|
public static Object getValue(String key, Map dataMap) { |
||||
|
try { |
||||
|
//公式为空直接就返回 |
||||
|
if (!StringUtil.hasValue(key)) { |
||||
|
return ""; |
||||
|
} |
||||
|
Object dataValue; |
||||
|
String isJson = "#json#"; |
||||
|
if (key.contains(isJson)) { |
||||
|
//进行第一次拆分,获取#json#前面的部分 |
||||
|
String[] keySplit = key.split(isJson); |
||||
|
String firstDataKey = keySplit[0]; |
||||
|
String[] firstDataKeySplit = firstDataKey.split(":"); |
||||
|
//取出前半部分对应的JSON数据并转换为JSONObject |
||||
|
String dataJson = (String) dataMap.get(firstDataKeySplit[0]); |
||||
|
JSONObject dataJsonObject = JSON.parseObject(dataJson); |
||||
|
//根据key的后半部分取出对应JSONObject中的值 |
||||
|
String firstDataKeyJson = (String) JSONPath.eval(dataJsonObject, firstDataKeySplit[1]); |
||||
|
String secDataKey = keySplit[1]; |
||||
|
JSONObject firstDataJsonObject = JSON.parseObject(firstDataKeyJson); |
||||
|
dataValue = JSONPath.eval(firstDataJsonObject, secDataKey); |
||||
|
return dataValue; |
||||
|
} |
||||
|
String[] keySplit = key.split(":"); |
||||
|
String jsonPath = keySplit[1]; |
||||
|
String dataJson = (String) dataMap.get(keySplit[0]); |
||||
|
JSONObject dataJsonObject = JSON.parseObject(dataJson); |
||||
|
dataValue = JSONPath.eval(dataJsonObject, jsonPath); |
||||
|
return dataValue; |
||||
|
} catch (Exception e) { |
||||
|
// TODO: handle exception |
||||
|
log.error("jsonpath公式取值异常,", e); |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
} |
@ -0,0 +1,182 @@ |
|||||
|
package com.bfd.crawl.ppthandler.util; |
||||
|
|
||||
|
import org.apache.poi.ss.usermodel.*; |
||||
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.io.FileInputStream; |
||||
|
import java.io.FileOutputStream; |
||||
|
import java.io.IOException; |
||||
|
import java.nio.file.Files; |
||||
|
import java.nio.file.Path; |
||||
|
import java.nio.file.Paths; |
||||
|
import java.nio.file.StandardCopyOption; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* @author jian.mao |
||||
|
* @date 2023年4月7日 |
||||
|
* @description excel解析工具类 |
||||
|
*/ |
||||
|
public class ExcelUtils { |
||||
|
/** |
||||
|
* excel解析 |
||||
|
* |
||||
|
* @param excel |
||||
|
* @return |
||||
|
*/ |
||||
|
public static Map<String, Object> parse(File excel) { |
||||
|
Map<String, Object> excelMap = new HashMap<String, Object>(16); |
||||
|
try { |
||||
|
FileInputStream file = new FileInputStream(excel); |
||||
|
// 使用工厂模式创建工作簿对象 |
||||
|
Workbook workbook = WorkbookFactory.create(file); |
||||
|
// 获取工作簿中工作表的数量 |
||||
|
int numberOfSheets = workbook.getNumberOfSheets(); |
||||
|
DataFormatter dataFormatter = new DataFormatter(); |
||||
|
// 遍历所有工作表 |
||||
|
for (int i = 0; i < numberOfSheets; i++) { |
||||
|
Sheet sheet = workbook.getSheetAt(i); |
||||
|
String key = sheet.getSheetName(); |
||||
|
//行码 |
||||
|
int rowNum = 0; |
||||
|
List<Map<String, String>> data = new ArrayList<Map<String, String>>(); |
||||
|
Map<Integer, String> titleHead = new HashMap<Integer, String>(16); |
||||
|
// 遍历所有行 |
||||
|
for (Row row : sheet) { |
||||
|
//单元格码 |
||||
|
int cellNum = 0; |
||||
|
//行内容存储 |
||||
|
Map<String, String> rowMap = new HashMap<String, String>(16); |
||||
|
// 遍历所有单元格 |
||||
|
if (rowNum == 0) { |
||||
|
for (Cell cell : row) { |
||||
|
String cellValue = dataFormatter.formatCellValue(cell); |
||||
|
titleHead.put(cellNum, cellValue); |
||||
|
cellNum++; |
||||
|
} |
||||
|
} else { |
||||
|
for (int j = 0; j < titleHead.size(); j++) { |
||||
|
String cellValue = dataFormatter.formatCellValue(row.getCell(j)); |
||||
|
rowMap.put(titleHead.get(cellNum), cellValue); |
||||
|
cellNum++; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (rowNum > 0) { |
||||
|
data.add(rowMap); |
||||
|
} |
||||
|
rowNum++; |
||||
|
} |
||||
|
excelMap.put(key, data); |
||||
|
} |
||||
|
// 关闭文件输入流和工作簿对象 |
||||
|
file.close(); |
||||
|
workbook.close(); |
||||
|
} catch (IOException e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
return excelMap; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 将List<Map<String, String>>写入Excel文件中 |
||||
|
* |
||||
|
* @param data 要写入Excel的数据,每个Map代表一行数据,Map的key为列名,value为单元格数据 |
||||
|
* @param excelFilePath Excel文件路径,包含文件名和扩展名 |
||||
|
* @param sheetName 工作表名称 |
||||
|
* @throws IOException 如果写入Excel文件时发生IO异常,则抛出该异常 |
||||
|
*/ |
||||
|
public static void write(List<Map<String, String>> data, String excelFilePath, String sheetName) throws IOException { |
||||
|
// 创建一个新的工作簿对象 |
||||
|
Workbook workbook = new XSSFWorkbook(); |
||||
|
// 创建一个新的工作表 |
||||
|
Sheet sheet = workbook.createSheet(sheetName); |
||||
|
// 行码 |
||||
|
int rowNum = 0; |
||||
|
// 写入列头 |
||||
|
Row headerRow = sheet.createRow(rowNum++); |
||||
|
int colNum = 0; |
||||
|
for (String key : data.get(0).keySet()) { |
||||
|
Cell cell = headerRow.createCell(colNum++); |
||||
|
cell.setCellValue(key); |
||||
|
} |
||||
|
// 写入数据 |
||||
|
for (Map<String, String> rowMap : data) { |
||||
|
Row row = sheet.createRow(rowNum++); |
||||
|
colNum = 0; |
||||
|
for (String key : rowMap.keySet()) { |
||||
|
Cell cell = row.createCell(colNum++); |
||||
|
try { |
||||
|
String s = rowMap.get(key); |
||||
|
if (s.length() > 30000) { |
||||
|
|
||||
|
cell.setCellValue(s.substring(0, 25000)); |
||||
|
} else { |
||||
|
cell.setCellValue(s); |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
System.out.println(key); |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
// 将数据写入文件 |
||||
|
FileOutputStream outputStream = new FileOutputStream(excelFilePath); |
||||
|
workbook.write(outputStream); |
||||
|
workbook.close(); |
||||
|
outputStream.close(); |
||||
|
} |
||||
|
|
||||
|
// public static void copyFile(String sourceFloder, String targetFileName) { |
||||
|
// File sourceFile = new File(sourceFloder); |
||||
|
// byte[] buffer = new byte[(int) sourceFile.length()]; |
||||
|
// try (InputStream inputStream = new FileInputStream(sourceFile)) { |
||||
|
// inputStream.read(buffer); |
||||
|
// } catch (IOException e) { |
||||
|
// e.printStackTrace(); |
||||
|
// return; |
||||
|
// } |
||||
|
// // 写入目标文件 |
||||
|
// File targetFile = new File(targetFileName); |
||||
|
// targetFile.mkdirs(); |
||||
|
// try (OutputStream outputStream = new FileOutputStream(targetFile)) { |
||||
|
// outputStream.write(buffer); |
||||
|
// } catch (IOException e) { |
||||
|
// e.printStackTrace(); |
||||
|
// return; |
||||
|
// } |
||||
|
// } |
||||
|
|
||||
|
public static void copyFile(String sourceFilePath) { |
||||
|
// 源文件和目标文件的路径 |
||||
|
String targetDrive = "F:"; |
||||
|
|
||||
|
try { |
||||
|
// 获取源文件和目标文件的路径信息 |
||||
|
Path sourcePath = Paths.get(sourceFilePath); |
||||
|
Path targetPath = Paths.get(targetDrive + sourcePath.toString().substring(2)); |
||||
|
|
||||
|
// 如果目标文件的父目录不存在,则创建该目录 |
||||
|
if (!targetPath.getParent().toFile().exists()) { |
||||
|
targetPath.getParent().toFile().mkdirs(); |
||||
|
} |
||||
|
|
||||
|
// 进行文件复制 |
||||
|
Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); |
||||
|
|
||||
|
System.out.println("Copied file: " + sourceFilePath + " -> " + targetPath); |
||||
|
} catch (IOException e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public static void main(String[] args) { |
||||
|
System.out.println(parse(new File("D:\\fromHanler\\xls\\b51484b213ed8fea61f5b99cbdc1490e.xlsx"))); |
||||
|
} |
||||
|
|
||||
|
} |
@ -0,0 +1,117 @@ |
|||||
|
package com.bfd.crawl.ppthandler.util; |
||||
|
|
||||
|
import com.alibaba.fastjson.JSON; |
||||
|
import okhttp3.*; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.io.FileOutputStream; |
||||
|
import java.io.IOException; |
||||
|
import java.io.InputStream; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:FileDownloader |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/8/1 16:31 |
||||
|
*/ |
||||
|
public class FileDownloader { |
||||
|
private static OkHttpClient okHttpClient; |
||||
|
|
||||
|
private static OkHttpClient getOkHttpClient() { |
||||
|
if (okHttpClient == null) { |
||||
|
okHttpClient = new OkHttpClient(); |
||||
|
} |
||||
|
return okHttpClient; |
||||
|
} |
||||
|
|
||||
|
public static void downloadFile(String url, File destination) throws IOException { |
||||
|
OkHttpClient client = getOkHttpClient(); |
||||
|
Request request = new Request.Builder() |
||||
|
.url(url) |
||||
|
.build(); |
||||
|
|
||||
|
try (Response response = client.newCall(request).execute()) { |
||||
|
if (!response.isSuccessful()) { |
||||
|
throw new IOException("Failed to download file: " + response); |
||||
|
} |
||||
|
|
||||
|
ResponseBody body = response.body(); |
||||
|
if (body == null) { |
||||
|
throw new IOException("Response body is null"); |
||||
|
} |
||||
|
if (!destination.getParentFile().exists()) { |
||||
|
|
||||
|
destination.getParentFile().mkdirs(); |
||||
|
} |
||||
|
try (InputStream inputStream = body.byteStream(); |
||||
|
FileOutputStream outputStream = new FileOutputStream(destination)) { |
||||
|
byte[] buffer = new byte[8192]; |
||||
|
int bytesRead; |
||||
|
while ((bytesRead = inputStream.read(buffer)) != -1) { |
||||
|
outputStream.write(buffer, 0, bytesRead); |
||||
|
} |
||||
|
outputStream.flush(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public static Map<String, String> uploadFile(String url, String filePath) throws Exception { |
||||
|
File file = new File(filePath); |
||||
|
|
||||
|
Map returnMap = new HashMap(32); |
||||
|
OkHttpClient client = getOkHttpClient(); |
||||
|
// 设置文件上传的媒体类型 |
||||
|
MediaType mediaType = MediaType.parse("application/octet-stream"); |
||||
|
// 创建请求体,将文件添加到请求体中 |
||||
|
RequestBody requestBody = RequestBody.create(mediaType, file); |
||||
|
|
||||
|
// 创建多部分请求体,用于上传文件 |
||||
|
MultipartBody multipartBody = new MultipartBody.Builder() |
||||
|
.setType(MultipartBody.FORM) |
||||
|
.addFormDataPart("file", file.getName(), requestBody) |
||||
|
.build(); |
||||
|
// 创建上传文件的请求 |
||||
|
Request request = new Request.Builder() |
||||
|
.url(url) |
||||
|
.post(multipartBody) |
||||
|
.build(); |
||||
|
|
||||
|
try (Response response = client.newCall(request).execute()) { |
||||
|
if (!response.isSuccessful()) { |
||||
|
throw new IOException("Failed to upload file: " + response); |
||||
|
} |
||||
|
String html = response.body().string(); |
||||
|
|
||||
|
try { |
||||
|
Map parse = (Map) JSON.parse(html); |
||||
|
Map data = (Map) parse.get("data"); |
||||
|
String domain = (String) data.get("domain"); |
||||
|
String src = (String) data.get("src"); |
||||
|
String fileUrl = domain.concat(src); |
||||
|
returnMap.put("fileUrl", fileUrl); |
||||
|
} catch (Exception e) { |
||||
|
returnMap.put("fileUrl", html); |
||||
|
} |
||||
|
// 处理上传成功的响应 |
||||
|
System.out.println("File uploaded successfully!"); |
||||
|
} |
||||
|
return returnMap; |
||||
|
} |
||||
|
|
||||
|
public static void deleteFile(String url, String md5) throws Exception { |
||||
|
OkHttpClient client = getOkHttpClient(); |
||||
|
url = url.concat("delete?md5=").concat(md5); |
||||
|
Request request = new Request.Builder() |
||||
|
.url(url) |
||||
|
.get() |
||||
|
.build(); |
||||
|
try (Response response = client.newCall(request).execute()) { |
||||
|
if (!response.isSuccessful()) { |
||||
|
throw new IOException("Failed to upload file: " + response); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
@ -0,0 +1,42 @@ |
|||||
|
package com.bfd.crawl.ppthandler.util; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.List; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:FileUtil |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/8/2 10:57 |
||||
|
*/ |
||||
|
public class FileUtil { |
||||
|
|
||||
|
public static void main(String[] args) { |
||||
|
System.out.println(traverseAndReturnFilePath("D:\\\\ocr\\\\305ce27d7a05770456fdc09d0b3044f7\\\\")); |
||||
|
} |
||||
|
|
||||
|
public static List<String> traverseAndReturnFilePath(String folderPath) { |
||||
|
List<String> fileList = new ArrayList<>(); |
||||
|
File folder = new File(folderPath); |
||||
|
// 检查文件夹是否存在并且是一个文件夹 |
||||
|
if (folder.exists() && folder.isDirectory()) { |
||||
|
// 获取文件夹中的所有文件和子文件夹 |
||||
|
File[] files = folder.listFiles(); |
||||
|
if (files != null) { |
||||
|
for (File file : files) { |
||||
|
if (file.isFile()) { |
||||
|
// 如果是文件,则输出全路径 |
||||
|
fileList.add(file.getAbsolutePath()); |
||||
|
} else if (file.isDirectory()) { |
||||
|
|
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} else { |
||||
|
System.out.println("指定的路径不是一个文件夹或文件夹不存在。"); |
||||
|
} |
||||
|
return fileList; |
||||
|
} |
||||
|
} |
@ -0,0 +1,59 @@ |
|||||
|
package com.bfd.crawl.ppthandler.util; |
||||
|
|
||||
|
import com.alibaba.fastjson.JSON; |
||||
|
import okhttp3.*; |
||||
|
|
||||
|
import java.util.Map; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:ocrUtil |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/8/1 16:38 |
||||
|
*/ |
||||
|
public class OcrUtil { |
||||
|
private static OkHttpClient okHttpClient; |
||||
|
|
||||
|
private static OkHttpClient getOkHttpClient() { |
||||
|
if (okHttpClient == null) { |
||||
|
okHttpClient = new OkHttpClient(); |
||||
|
} |
||||
|
return okHttpClient; |
||||
|
} |
||||
|
|
||||
|
public static String doOcr(String url,String ocrApi) { |
||||
|
String text = ""; |
||||
|
int reTryTimes = 3; |
||||
|
for (int i = 0; i < reTryTimes; i++) { |
||||
|
int okCode = 200; |
||||
|
OkHttpClient client = getOkHttpClient(); |
||||
|
OkHttpClient.Builder builder = client.newBuilder().writeTimeout(600, TimeUnit.SECONDS).connectTimeout(600, TimeUnit.SECONDS).readTimeout(600, TimeUnit.SECONDS); |
||||
|
client = builder.build(); |
||||
|
MediaType mediaType = MediaType.parse("application/json"); |
||||
|
RequestBody body = RequestBody.create(mediaType, "{\"id\":\"\",\"url\":\"" + url + "\"}"); |
||||
|
Request request = new Request.Builder() |
||||
|
.url(ocrApi) |
||||
|
.method("POST", body) |
||||
|
.addHeader("Content-Type", "application/json") |
||||
|
.build(); |
||||
|
try { |
||||
|
Response response = client.newCall(request).execute(); |
||||
|
String html = response.body().string(); |
||||
|
Map dataMap = (Map) JSON.parse(html); |
||||
|
int code = (int) dataMap.get("code"); |
||||
|
if (code == okCode) { |
||||
|
text = (String) dataMap.get("text"); |
||||
|
} |
||||
|
if (StringUtil.hasValue(text)) { |
||||
|
break; |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return text; |
||||
|
} |
||||
|
} |
@ -0,0 +1,23 @@ |
|||||
|
package com.bfd.crawl.ppthandler.util; |
||||
|
|
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
|
||||
|
import java.util.Locale; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:OsUtil |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/4/23 9:40 |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public class OsUtil { |
||||
|
public static Boolean isWindows() { |
||||
|
String windows = "windows"; |
||||
|
String osName = System.getProperty("os.name").toLowerCase(Locale.ROOT); |
||||
|
log.info("osName = " + osName); |
||||
|
return osName.contains(windows); |
||||
|
} |
||||
|
|
||||
|
} |
@ -0,0 +1,124 @@ |
|||||
|
package com.bfd.crawl.ppthandler.util; |
||||
|
|
||||
|
import org.apache.poi.hslf.usermodel.HSLFShape; |
||||
|
import org.apache.poi.hslf.usermodel.HSLFSlide; |
||||
|
import org.apache.poi.hslf.usermodel.HSLFSlideShow; |
||||
|
import org.apache.poi.hslf.usermodel.HSLFTextShape; |
||||
|
import org.apache.poi.xslf.usermodel.*; |
||||
|
|
||||
|
import java.io.FileInputStream; |
||||
|
import java.io.IOException; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.List; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:PptUtil |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2024/3/25 16:14 |
||||
|
*/ |
||||
|
public class PptUtil { |
||||
|
|
||||
|
|
||||
|
public static String parsepptx(String filePath) { |
||||
|
StringBuilder dataStringsb = new StringBuilder(); |
||||
|
XMLSlideShow ppt = null; |
||||
|
try { |
||||
|
// PPT类 |
||||
|
ppt = new XMLSlideShow(new FileInputStream(filePath)); |
||||
|
// 获取PPT中的所有幻灯片 |
||||
|
// List<XSLFSlide> slides = ppt.getSlides(); |
||||
|
// 遍历幻灯片 |
||||
|
for (int i = 0; i < ppt.getSlides().size(); i++) { |
||||
|
|
||||
|
//拿到第i页的PPT |
||||
|
XSLFSlide slides = ppt.getSlides().get(i); |
||||
|
System.out.println("第" + (i + 1) + "页"); |
||||
|
//注释的for循环是获取所以PPT的内容 |
||||
|
// for (XSLFSlide slide : slides) { |
||||
|
// 获取幻灯片中的所有图形 |
||||
|
List<XSLFShape> shapes = slides.getShapes(); |
||||
|
// 遍历PPT的图形 |
||||
|
for (XSLFShape shape : shapes) { |
||||
|
// 判断该图形类是否是文本框类 |
||||
|
if (shape instanceof XSLFTextShape) { |
||||
|
// 将图像类强制装换成文本框类 |
||||
|
XSLFTextShape ts = (XSLFTextShape) shape; |
||||
|
// 获取文本框内的文字 |
||||
|
String str = ts.getText(); |
||||
|
dataStringsb.append(str); |
||||
|
} |
||||
|
// 判断该图形类是否是表格类 |
||||
|
if (shape instanceof XSLFTable) { |
||||
|
// 将图像类强制装换成表格类 |
||||
|
XSLFTable table = (XSLFTable) shape; |
||||
|
// 获取表格中的所有行 |
||||
|
List<XSLFTableRow> rows = table.getRows(); |
||||
|
for (XSLFTableRow tr : rows) { |
||||
|
// 获取行中的所有单元格 |
||||
|
List<XSLFTableCell> cells = tr.getCells(); |
||||
|
for (XSLFTableCell tc : cells) { |
||||
|
// 获取单元格内的文字 |
||||
|
String str = tc.getText(); |
||||
|
dataStringsb.append(str); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
// 判断该图形类是否是图片框类chr |
||||
|
if (shape instanceof XSLFPictureShape) { |
||||
|
// 将图像类强制装换成图片框类 |
||||
|
XSLFPictureShape ps = (XSLFPictureShape) shape; |
||||
|
// 获取图片的字节码数据(可以利用输出流将该图片保存到硬盘里) |
||||
|
byte[] pictureData = ps.getPictureData().getData(); |
||||
|
// System.out.println("图片信息:" + pictureData); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
// } |
||||
|
} catch (Exception e) { |
||||
|
e.printStackTrace(); |
||||
|
} finally { |
||||
|
if (ppt != null) { |
||||
|
|
||||
|
try { |
||||
|
// 保存完之后要对PPT进行关闭操作 |
||||
|
ppt.close(); |
||||
|
} catch (IOException e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
} |
||||
|
return dataStringsb.toString(); |
||||
|
} |
||||
|
|
||||
|
public static String parsePpt(String filePath) { |
||||
|
StringBuilder dataStringsb = new StringBuilder(); |
||||
|
HSLFSlideShow ppt = null; |
||||
|
try { |
||||
|
ppt = new HSLFSlideShow(new FileInputStream(filePath)); |
||||
|
for (HSLFSlide slide : ppt.getSlides()) { |
||||
|
for (HSLFShape shape : slide.getShapes()) { |
||||
|
if (shape instanceof HSLFTextShape) { |
||||
|
HSLFTextShape textShape = (HSLFTextShape) shape; |
||||
|
dataStringsb.append(textShape.getText()).append("\n"); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
e.printStackTrace(); |
||||
|
} finally { |
||||
|
if (ppt != null) { |
||||
|
try { |
||||
|
ppt.close(); |
||||
|
} catch (IOException e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return dataStringsb.toString(); |
||||
|
} |
||||
|
} |
@ -0,0 +1,19 @@ |
|||||
|
package com.bfd.crawl.ppthandler.util; |
||||
|
|
||||
|
import java.util.concurrent.LinkedBlockingDeque; |
||||
|
|
||||
|
/** |
||||
|
* @author:jinming |
||||
|
* @className:QueueUtil |
||||
|
* @version:1.0 |
||||
|
* @description: |
||||
|
* @Date:2023/7/13 15:00 |
||||
|
*/ |
||||
|
public class QueueUtil { |
||||
|
|
||||
|
|
||||
|
public static LinkedBlockingDeque<String> taskQueue = new LinkedBlockingDeque<String>(); |
||||
|
|
||||
|
public static LinkedBlockingDeque<String> sendQueue = new LinkedBlockingDeque<String>(); |
||||
|
|
||||
|
} |
@ -0,0 +1,94 @@ |
|||||
|
package com.bfd.crawl.ppthandler.util; |
||||
|
|
||||
|
|
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
|
||||
|
import java.security.MessageDigest; |
||||
|
import java.util.HashSet; |
||||
|
import java.util.Set; |
||||
|
import java.util.regex.Matcher; |
||||
|
import java.util.regex.Pattern; |
||||
|
|
||||
|
/** |
||||
|
* @author jinming |
||||
|
* @version 1.0 |
||||
|
* @className StringUtile |
||||
|
* @Date 2022/1/21 11:46 |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public class StringUtil { |
||||
|
public static boolean hasValue(String str) { |
||||
|
return str != null && !"".equals(str.trim()); |
||||
|
} |
||||
|
|
||||
|
public static String getRegexGroup(String regex, String str, int id) { |
||||
|
String resultStr = ""; |
||||
|
if (hasValue(str)) { |
||||
|
Pattern p = Pattern.compile(regex); |
||||
|
Matcher m = p.matcher(str); |
||||
|
if (m.find()) { |
||||
|
resultStr = m.group(id); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if ("".equals(resultStr)) { |
||||
|
} |
||||
|
|
||||
|
return resultStr; |
||||
|
} |
||||
|
|
||||
|
public static Set<String> getEmailAddress(String message) { |
||||
|
Set<String> emailList = new HashSet<>(); |
||||
|
Pattern pattern = Pattern.compile("\\w+\\.?\\w+\\@\\w+\\.\\w+"); |
||||
|
Matcher m = pattern.matcher(message); |
||||
|
while (m.find()) { |
||||
|
emailList.add(m.group(0)); |
||||
|
} |
||||
|
return emailList; |
||||
|
} |
||||
|
public static String getMd5(String string) { |
||||
|
try { |
||||
|
MessageDigest md5 = MessageDigest.getInstance("MD5"); |
||||
|
byte[] bs = md5.digest(string.getBytes("UTF-8")); |
||||
|
StringBuilder sb = new StringBuilder(40); |
||||
|
for (byte x : bs) { |
||||
|
if ((x & 0xff) >> 4 == 0) { |
||||
|
sb.append("0").append(Integer.toHexString(x & 0xff)); |
||||
|
} else { |
||||
|
sb.append(Integer.toHexString(x & 0xff)); |
||||
|
} |
||||
|
} |
||||
|
return sb.toString(); |
||||
|
} catch (Exception e) { |
||||
|
//LOG.error("获取md5异常", e); |
||||
|
return "nceaform" + System.currentTimeMillis(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public static String removeAllHtmlTags(String str) { |
||||
|
return hasValue(str) ? str.replaceAll("<[^<>]+?>", "") : ""; |
||||
|
} |
||||
|
|
||||
|
public static String getRegexGroup(Pattern regex, String str, int id) { |
||||
|
String resultStr = ""; |
||||
|
if (hasValue(str)) { |
||||
|
Matcher m = regex.matcher(str); |
||||
|
if (m.find()) { |
||||
|
resultStr = m.group(id); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if ("".equals(resultStr)) { |
||||
|
log.error(regex + " parser error!"); |
||||
|
} |
||||
|
|
||||
|
return resultStr; |
||||
|
} |
||||
|
|
||||
|
public static String getStrByPattern(String str, String regex) { |
||||
|
Pattern pattern = Pattern.compile(regex); |
||||
|
Matcher m = pattern.matcher(str); |
||||
|
return m.find() ? m.group(0) : ""; |
||||
|
} |
||||
|
|
||||
|
} |
@ -0,0 +1,47 @@ |
|||||
|
server: |
||||
|
port: 7082 |
||||
|
spring: |
||||
|
application: |
||||
|
name: 幻灯片处理 |
||||
|
boot: |
||||
|
admin: |
||||
|
client: |
||||
|
health: |
||||
|
timeout: 10s |
||||
|
url: http://172.16.12.55:8001 |
||||
|
instance: |
||||
|
service-base-url: http://172.16.12.55:7088 |
||||
|
kafka: |
||||
|
bootstrap-servers: 172.26.28.30:9092 |
||||
|
producer: |
||||
|
retries: 3 |
||||
|
acks: all |
||||
|
batch-size: 4096 |
||||
|
buffer-memory: 102476800 |
||||
|
key-serializer: org.apache.kafka.common.serialization.StringSerializer |
||||
|
value-serializer: org.apache.kafka.common.serialization.StringSerializer |
||||
|
|
||||
|
|
||||
|
logging: |
||||
|
file: |
||||
|
path: ./logs |
||||
|
|
||||
|
management: |
||||
|
endpoints: |
||||
|
web: |
||||
|
exposure: |
||||
|
include: "*" |
||||
|
endpoint: |
||||
|
health: |
||||
|
show-details: always |
||||
|
|
||||
|
send: |
||||
|
topic: analyze0912 |
||||
|
|
||||
|
file: |
||||
|
path: D:\\fromHanler\\ |
||||
|
|
||||
|
thread: |
||||
|
handler: 1 |
||||
|
send: 1 |
||||
|
|
@ -0,0 +1,36 @@ |
|||||
|
<configuration> |
||||
|
<!-- 属性文件:在properties文件中找到对应的配置项 --> |
||||
|
<springProperty scope="context" name="logging.file.path" source="logging.file.path"/> |
||||
|
<springProperty scope="context" name="logging.level" source="logging.level"/> |
||||
|
<!-- 默认的控制台日志输出,一般生产环境都是后台启动,这个没太大作用 --> |
||||
|
<appender name="STDOUT" |
||||
|
class="ch.qos.logback.core.ConsoleAppender"> |
||||
|
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> |
||||
|
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</pattern> |
||||
|
</encoder> |
||||
|
</appender> |
||||
|
|
||||
|
<appender name="GLMAPPER-LOGGERONE" |
||||
|
class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
|
<append>true</append> |
||||
|
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||
|
<level>${logging.level}</level> |
||||
|
</filter> |
||||
|
<file> |
||||
|
${logging.file.path}/formHandler.log |
||||
|
</file> |
||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> |
||||
|
<FileNamePattern>${logging.file.path}/formHandler.log.%d{yyyy-MM-dd}</FileNamePattern> |
||||
|
<MaxHistory>3</MaxHistory> |
||||
|
</rollingPolicy> |
||||
|
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> |
||||
|
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</pattern> |
||||
|
<charset>UTF-8</charset> |
||||
|
</encoder> |
||||
|
</appender> |
||||
|
|
||||
|
<root level="info"> |
||||
|
<appender-ref ref="GLMAPPER-LOGGERONE"/> |
||||
|
<appender-ref ref="STDOUT"/> |
||||
|
</root> |
||||
|
</configuration> |
Write
Preview
Loading…
Cancel
Save
Reference in new issue