commit
cb8c11dfdd
24 changed files with 1483 additions and 0 deletions
-
33.gitignore
-
1README.md
-
138pom.xml
-
13src/main/java/com/bfd/crawl/ppthandler/PptHanlerApplication.java
-
60src/main/java/com/bfd/crawl/ppthandler/bean/ResponsePo.java
-
48src/main/java/com/bfd/crawl/ppthandler/config/AsyncThreadConfiguration.java
-
27src/main/java/com/bfd/crawl/ppthandler/config/Constant.java
-
46src/main/java/com/bfd/crawl/ppthandler/controller/PptHandlerController.java
-
32src/main/java/com/bfd/crawl/ppthandler/enums/ResponseCode.java
-
121src/main/java/com/bfd/crawl/ppthandler/service/HandlerService.java
-
54src/main/java/com/bfd/crawl/ppthandler/service/SendService.java
-
63src/main/java/com/bfd/crawl/ppthandler/service/StartServcie.java
-
44src/main/java/com/bfd/crawl/ppthandler/util/CsvUtil.java
-
60src/main/java/com/bfd/crawl/ppthandler/util/DataUtil.java
-
182src/main/java/com/bfd/crawl/ppthandler/util/ExcelUtils.java
-
117src/main/java/com/bfd/crawl/ppthandler/util/FileDownloader.java
-
42src/main/java/com/bfd/crawl/ppthandler/util/FileUtil.java
-
59src/main/java/com/bfd/crawl/ppthandler/util/OcrUtil.java
-
23src/main/java/com/bfd/crawl/ppthandler/util/OsUtil.java
-
124src/main/java/com/bfd/crawl/ppthandler/util/PptUtil.java
-
19src/main/java/com/bfd/crawl/ppthandler/util/QueueUtil.java
-
94src/main/java/com/bfd/crawl/ppthandler/util/StringUtil.java
-
47src/main/resources/application.yml
-
36src/main/resources/logback-spring.xml
@ -0,0 +1,33 @@ |
|||
HELP.md |
|||
target/ |
|||
!.mvn/wrapper/maven-wrapper.jar |
|||
!**/src/main/**/target/ |
|||
!**/src/test/**/target/ |
|||
|
|||
### STS ### |
|||
.apt_generated |
|||
.classpath |
|||
.factorypath |
|||
.project |
|||
.settings |
|||
.springBeans |
|||
.sts4-cache |
|||
|
|||
### IntelliJ IDEA ### |
|||
.idea |
|||
*.iws |
|||
*.iml |
|||
*.ipr |
|||
|
|||
### NetBeans ### |
|||
/nbproject/private/ |
|||
/nbbuild/ |
|||
/dist/ |
|||
/nbdist/ |
|||
/.nb-gradle/ |
|||
build/ |
|||
!**/src/main/**/build/ |
|||
!**/src/test/**/build/ |
|||
|
|||
### VS Code ### |
|||
.vscode/ |
@ -0,0 +1 @@ |
|||
ppt 解析应用 |
@ -0,0 +1,138 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> |
|||
<modelVersion>4.0.0</modelVersion> |
|||
<parent> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter-parent</artifactId> |
|||
<version>2.2.4.RELEASE</version> |
|||
<relativePath/> <!-- lookup parent from repository --> |
|||
</parent> |
|||
<groupId>com.bfd.crawl</groupId> |
|||
<artifactId>pptHandler</artifactId> |
|||
<version>0.0.1-SNAPSHOT</version> |
|||
<name>pptHandler</name> |
|||
<description>pptHandler</description> |
|||
<properties> |
|||
<java.version>8</java.version> |
|||
</properties> |
|||
|
|||
|
|||
<dependencies> |
|||
<!-- https://mvnrepository.com/artifact/de.codecentric/spring-boot-admin-starter-client --> |
|||
<dependency> |
|||
<groupId>de.codecentric</groupId> |
|||
<artifactId>spring-boot-admin-client</artifactId> |
|||
<version>2.2.4</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.apache.commons</groupId> |
|||
<artifactId>commons-csv</artifactId> |
|||
<version>1.10.0</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.kafka</groupId> |
|||
<artifactId>spring-kafka</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter-web</artifactId> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-devtools</artifactId> |
|||
<scope>runtime</scope> |
|||
<optional>true</optional> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.projectlombok</groupId> |
|||
<artifactId>lombok</artifactId> |
|||
<optional>true</optional> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-starter-test</artifactId> |
|||
<scope>test</scope> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.apache.pdfbox</groupId> |
|||
<artifactId>pdfbox</artifactId> |
|||
<version>2.0.28</version> |
|||
</dependency> |
|||
<!-- Apache POI --> |
|||
<dependency> |
|||
<groupId>org.apache.poi</groupId> |
|||
<artifactId>poi-scratchpad</artifactId> |
|||
<version>5.2.0</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.apache.poi</groupId> |
|||
<artifactId>poi</artifactId> |
|||
<version>5.2.0</version> |
|||
<exclusions> |
|||
<exclusion> |
|||
<groupId>org.apache.logging.log4j</groupId> |
|||
<artifactId>log4j-api</artifactId> |
|||
</exclusion> |
|||
</exclusions> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.apache.logging.log4j</groupId> |
|||
<artifactId>log4j-api</artifactId> |
|||
<version>2.17.1</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.apache.poi</groupId> |
|||
<artifactId>poi-ooxml</artifactId> |
|||
<version>5.2.0</version> |
|||
</dependency> |
|||
<!--JSON--> |
|||
<dependency> |
|||
<groupId>com.alibaba</groupId> |
|||
<artifactId>fastjson</artifactId> |
|||
<version>2.0.17</version> |
|||
</dependency> |
|||
<!--OKHTTP--> |
|||
<dependency> |
|||
<groupId>com.squareup.okhttp3</groupId> |
|||
<artifactId>okhttp</artifactId> |
|||
<version>3.9.1</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>com.google.code.gson</groupId> |
|||
<artifactId>gson</artifactId> |
|||
<version>2.8.8</version> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.apache.kafka</groupId> |
|||
<artifactId>kafka-clients</artifactId> |
|||
<version>2.3.1</version> <!--根据您正在使用的Kafka版本选择合适的版本号--> |
|||
</dependency> |
|||
<dependency> |
|||
<groupId>org.springframework.kafka</groupId> |
|||
<artifactId>spring-kafka-test</artifactId> |
|||
<scope>test</scope> |
|||
</dependency> |
|||
</dependencies> |
|||
<build> |
|||
<plugins> |
|||
<plugin> |
|||
<groupId>org.springframework.boot</groupId> |
|||
<artifactId>spring-boot-maven-plugin</artifactId> |
|||
<configuration> |
|||
<excludes> |
|||
<exclude> |
|||
<groupId>org.projectlombok</groupId> |
|||
<artifactId>lombok</artifactId> |
|||
</exclude> |
|||
</excludes> |
|||
</configuration> |
|||
</plugin> |
|||
</plugins> |
|||
</build> |
|||
|
|||
</project> |
@ -0,0 +1,13 @@ |
|||
package com.bfd.crawl.ppthandler; |
|||
|
|||
import org.springframework.boot.SpringApplication; |
|||
import org.springframework.boot.autoconfigure.SpringBootApplication; |
|||
|
|||
@SpringBootApplication |
|||
public class PptHanlerApplication { |
|||
|
|||
public static void main(String[] args) { |
|||
SpringApplication.run(PptHanlerApplication.class, args); |
|||
} |
|||
|
|||
} |
@ -0,0 +1,60 @@ |
|||
package com.bfd.crawl.ppthandler.bean; |
|||
|
|||
|
|||
|
|||
import com.bfd.crawl.ppthandler.enums.ResponseCode; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:ResponsePo |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/4/3 17:23 |
|||
*/ |
|||
@Data |
|||
@NoArgsConstructor |
|||
@AllArgsConstructor |
|||
public class ResponsePo { |
|||
/** |
|||
* 响应码 |
|||
*/ |
|||
private int code; |
|||
|
|||
/** |
|||
* 正常放 返回数据 的JSON串 |
|||
*/ |
|||
private Object data; |
|||
|
|||
/** |
|||
* 提示消息 |
|||
*/ |
|||
private String message; |
|||
|
|||
public static ResponsePo success() { |
|||
return setStatus(ResponseCode.SUCCESS.getCode(), ResponseCode.SUCCESS.getMessage()); |
|||
} |
|||
|
|||
public static ResponsePo error() { |
|||
return setStatus(ResponseCode.FAILURE.getCode(), ResponseCode.FAILURE.getMessage()); |
|||
} |
|||
|
|||
public static ResponsePo setStatus(int code, String message) { |
|||
ResponsePo resultBean = new ResponsePo(); |
|||
resultBean.code = code; |
|||
resultBean.message = message; |
|||
return resultBean; |
|||
} |
|||
public ResponsePo(int code, String message) { |
|||
this.code = code; |
|||
this.message = message; |
|||
this.data = data; |
|||
} |
|||
public ResponsePo(ResponseCode responseCode){ |
|||
this.code = responseCode.getCode(); |
|||
this.message = responseCode.getMessage(); |
|||
this.data = data; |
|||
} |
|||
} |
@ -0,0 +1,48 @@ |
|||
package com.bfd.crawl.ppthandler.config; |
|||
|
|||
|
|||
import org.springframework.context.annotation.Bean; |
|||
import org.springframework.context.annotation.Configuration; |
|||
import org.springframework.scheduling.annotation.EnableAsync; |
|||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; |
|||
|
|||
import java.util.concurrent.Executor; |
|||
|
|||
/** |
|||
* @author jinming |
|||
* @version 1.0 |
|||
* @className AsyncThreadConfiguration |
|||
* @Date 2022/2/17 18:37 |
|||
*/ |
|||
@Configuration |
|||
@EnableAsync |
|||
public class AsyncThreadConfiguration { |
|||
@Bean |
|||
public Executor asyncExecutor() { |
|||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); |
|||
// 核心线程数 |
|||
executor.setCorePoolSize(500); |
|||
// 并发线程的数量限制为2 |
|||
executor.setMaxPoolSize(500); |
|||
// 线程队列 |
|||
executor.setQueueCapacity(500); |
|||
executor.setThreadNamePrefix("handlerData-"); |
|||
executor.initialize(); |
|||
executor.setWaitForTasksToCompleteOnShutdown(true); |
|||
return executor; |
|||
} |
|||
@Bean |
|||
public Executor sendExecutor() { |
|||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); |
|||
// 核心线程数 |
|||
executor.setCorePoolSize(500); |
|||
// 并发线程的数量限制为2 |
|||
executor.setMaxPoolSize(500); |
|||
// 线程队列 |
|||
executor.setQueueCapacity(500); |
|||
executor.setThreadNamePrefix("sendData-"); |
|||
executor.initialize(); |
|||
executor.setWaitForTasksToCompleteOnShutdown(true); |
|||
return executor; |
|||
} |
|||
} |
@ -0,0 +1,27 @@ |
|||
package com.bfd.crawl.ppthandler.config; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:Constant |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/8/16 15:26 |
|||
*/ |
|||
public class Constant { |
|||
/** |
|||
* |
|||
*/ |
|||
public final static String IS_XLS = "xls"; |
|||
|
|||
/** |
|||
* |
|||
*/ |
|||
public final static String IS_CSV = "csv"; |
|||
|
|||
/** |
|||
* |
|||
*/ |
|||
public final static String ALL = "*"; |
|||
|
|||
|
|||
} |
@ -0,0 +1,46 @@ |
|||
package com.bfd.crawl.ppthandler.controller; |
|||
|
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
|
|||
import com.bfd.crawl.ppthandler.bean.ResponsePo; |
|||
import com.bfd.crawl.ppthandler.enums.ResponseCode; |
|||
import com.bfd.crawl.ppthandler.util.QueueUtil; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.web.bind.annotation.PostMapping; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.RestController; |
|||
|
|||
import java.util.Map; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:DataFilterController |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/7/26 11:21 |
|||
*/ |
|||
@RestController |
|||
@RequestMapping("/handlerdata") |
|||
@Slf4j |
|||
public class PptHandlerController { |
|||
@PostMapping("/ppthandler") |
|||
public ResponsePo documentFeedback(@RequestBody String dataJson) { |
|||
|
|||
ResponsePo responsePo = ResponsePo.success(); |
|||
try { |
|||
Map parse = (Map) JSON.parse(dataJson); |
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
log.error("请求格式发生异常" + e.getMessage()); |
|||
responsePo.setCode(ResponseCode.FAILURE.getCode()); |
|||
responsePo.setMessage(ResponseCode.FAILURE.getMessage()); |
|||
return responsePo; |
|||
} |
|||
log.info("新增任务:" + dataJson); |
|||
QueueUtil.taskQueue.add(dataJson); |
|||
|
|||
return responsePo; |
|||
} |
|||
} |
@ -0,0 +1,32 @@ |
|||
package com.bfd.crawl.ppthandler.enums; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:ResponseCodeEnum |
|||
* @version:1.0 |
|||
* @description:响应结果码枚举类 |
|||
* @Date:2023/2/28 11:40 |
|||
*/ |
|||
public enum ResponseCode { |
|||
//返回结果码枚举类 |
|||
SUCCESS(200, "操作成功"), |
|||
FAILURE(400, "参数错误"), |
|||
INTERNAL_SERVER_ERROR(500, "服务器内部错误"), |
|||
TYPE_NOT_SUPPORT(601,"文件类型不支持"); |
|||
|
|||
private int code; |
|||
private String message; |
|||
|
|||
ResponseCode(int code, String message) { |
|||
this.code = code; |
|||
this.message = message; |
|||
} |
|||
|
|||
public int getCode() { |
|||
return code; |
|||
} |
|||
|
|||
public String getMessage() { |
|||
return message; |
|||
} |
|||
} |
@ -0,0 +1,121 @@ |
|||
package com.bfd.crawl.ppthandler.service; |
|||
|
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
import com.bfd.crawl.ppthandler.config.Constant; |
|||
import com.bfd.crawl.ppthandler.util.*; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.scheduling.annotation.Async; |
|||
import org.springframework.stereotype.Service; |
|||
|
|||
import java.io.File; |
|||
import java.io.IOException; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:HandlerService |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/8/1 16:05 |
|||
*/ |
|||
@Service |
|||
@Slf4j |
|||
public class HandlerService { |
|||
@Value("${file.path}") |
|||
private String downloadFilePath; |
|||
|
|||
@Async("asyncExecutor") |
|||
void run() { |
|||
while (true) { |
|||
try { |
|||
if (QueueUtil.taskQueue.size() > 0) { |
|||
String dataJson = null; |
|||
try { |
|||
dataJson = QueueUtil.taskQueue.take(); |
|||
} catch (InterruptedException e) { |
|||
e.printStackTrace(); |
|||
continue; |
|||
} |
|||
String errorMessage = ""; |
|||
Map parse = (Map) JSON.parse(dataJson); |
|||
Map dataMap = (Map) parse.get("data"); |
|||
int id = (int) parse.get("id"); |
|||
Map admin = (Map) parse.get("input"); |
|||
|
|||
String key = (String) admin.get("fileUrl"); |
|||
String fileUrl = (String) DataUtil.getValue(key, dataMap); |
|||
boolean windows = OsUtil.isWindows(); |
|||
Map output = (Map) parse.get("output"); |
|||
// List<String> sheetName = (List<String>) admin.get("sheetName"); |
|||
//fieldType:自定义输出字段: 0 关闭,1-开启,如果开启则拼接form到output里(如果关闭,则取默认的output拼接) |
|||
String fileType = StringUtil.getStrByPattern(fileUrl, "\\.[^.\\\\/:*?\"<>|\\r\\n]+$"); |
|||
log.info("任务:" + id + "的文件类型为" + fileType); |
|||
String fileName = StringUtil.getMd5(fileUrl); |
|||
//定义ppt的下载路径 |
|||
String pptDir = null; |
|||
try { |
|||
pptDir = downloadFilePath.concat(windows ? "\\ppt\\" : "/ppt/").concat(fileName).concat(fileType); |
|||
FileDownloader.downloadFile(fileUrl, new File(pptDir)); |
|||
} catch (IOException e) { |
|||
e.printStackTrace(); |
|||
errorMessage = "文件下载失败"; |
|||
} |
|||
String pptParse = null; |
|||
String pptx = "pptx"; |
|||
try { |
|||
if (fileType.contains(pptx)) { |
|||
pptParse = PptUtil.parsepptx(pptDir); |
|||
} else { |
|||
pptParse = PptUtil.parsePpt(pptDir); |
|||
} |
|||
|
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
errorMessage = "文件解析失败"; |
|||
} |
|||
|
|||
if (!StringUtil.hasValue(errorMessage)) { |
|||
Map result = new HashMap(32); |
|||
Map resultsMap = new HashMap(32); |
|||
resultsMap.put("result", pptParse); |
|||
resultsMap.put("isLast", 1); |
|||
result.put("status", 1); |
|||
result.put("message", "成功"); |
|||
result.put("results", JSON.toJSONString(resultsMap)); |
|||
parse.put("result", result); |
|||
String message = JSON.toJSONString(parse); |
|||
QueueUtil.sendQueue.put(message); |
|||
continue; |
|||
} |
|||
Map result = new HashMap(32); |
|||
Map resultsMap = new HashMap(32); |
|||
resultsMap.put("content", errorMessage); |
|||
resultsMap.put("isLast", 1); |
|||
result.put("results", JSON.toJSONString(resultsMap)); |
|||
result.put("status", 2); |
|||
result.put("message", errorMessage); |
|||
parse.put("result", result); |
|||
String message = JSON.toJSONString(parse); |
|||
QueueUtil.sendQueue.put(message); |
|||
} else { |
|||
log.info("任务队列为空,休眠3秒"); |
|||
try { |
|||
Thread.sleep(3000); |
|||
} catch (InterruptedException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
} catch (Throwable e) { |
|||
e.printStackTrace(); |
|||
log.error("工作线程发生异常" + e.getMessage()); |
|||
} |
|||
} |
|||
} |
|||
|
|||
|
|||
} |
@ -0,0 +1,54 @@ |
|||
package com.bfd.crawl.ppthandler.service; |
|||
|
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
import com.bfd.crawl.ppthandler.util.QueueUtil; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.kafka.core.KafkaTemplate; |
|||
import org.springframework.scheduling.annotation.Async; |
|||
import org.springframework.stereotype.Service; |
|||
|
|||
import javax.annotation.Resource; |
|||
import java.util.Map; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:SendService |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/7/31 17:53 |
|||
*/ |
|||
@Slf4j |
|||
@Service |
|||
public class SendService { |
|||
@Value("${send.topic}") |
|||
private String topic; |
|||
|
|||
@Resource |
|||
private KafkaTemplate kafkaTemplate; |
|||
|
|||
@Async("sendExecutor") |
|||
void sendToKafka() { |
|||
while (true) { |
|||
if (QueueUtil.sendQueue.size() > 0) { |
|||
try { |
|||
String message = QueueUtil.sendQueue.take(); |
|||
Map parse = (Map) JSON.parse(message); |
|||
String id = parse.get("id").toString(); |
|||
log.info("ID:" + id + "\t" + "数据已发出"); |
|||
kafkaTemplate.send(topic, message); |
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} else { |
|||
log.info("任务队列为空,休眠3秒"); |
|||
try { |
|||
Thread.sleep(3000); |
|||
} catch (InterruptedException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,63 @@ |
|||
package com.bfd.crawl.ppthandler.service; |
|||
|
|||
|
|||
import com.bfd.crawl.ppthandler.util.QueueUtil; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.ApplicationArguments; |
|||
import org.springframework.boot.ApplicationRunner; |
|||
import org.springframework.stereotype.Service; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:StartServcie |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/7/31 17:14 |
|||
*/ |
|||
@Service |
|||
@Slf4j |
|||
public class StartServcie implements ApplicationRunner { |
|||
@Value("${thread.handler}") |
|||
private int handlerNumber; |
|||
@Value("${thread.send}") |
|||
private int sendNumber; |
|||
|
|||
@Autowired |
|||
private HandlerService handlerService; |
|||
@Autowired |
|||
private SendService sendService; |
|||
|
|||
@Override |
|||
public void run(ApplicationArguments args) throws Exception { |
|||
for (int i = 0; i < handlerNumber; i++) { |
|||
log.info("处理服务线程" + i + "已启动 "); |
|||
handlerService.run(); |
|||
} |
|||
for (int i = 0; i < sendNumber; i++) { |
|||
log.info("发送服务线程" + i + "已启动 "); |
|||
sendService.sendToKafka(); |
|||
} |
|||
Runnable myRunnable = new Runnable() { |
|||
@Override |
|||
public void run() { |
|||
while (true) { |
|||
log.info("任务队列长度为" + QueueUtil.taskQueue.size()); |
|||
log.info("发送队列长度为" + QueueUtil.sendQueue.size()); |
|||
try { |
|||
Thread.sleep(10000); |
|||
} catch (InterruptedException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
} |
|||
}; |
|||
// 创建一个新的线程,并将Runnable对象传递给Thread构造函数 |
|||
Thread myThread = new Thread(myRunnable); |
|||
// 启动线程 |
|||
myThread.start(); |
|||
|
|||
|
|||
} |
|||
} |
@ -0,0 +1,44 @@ |
|||
package com.bfd.crawl.ppthandler.util; |
|||
|
|||
import org.apache.commons.csv.CSVFormat; |
|||
import org.apache.commons.csv.CSVParser; |
|||
import org.apache.commons.csv.CSVRecord; |
|||
|
|||
import java.io.FileInputStream; |
|||
import java.io.InputStreamReader; |
|||
import java.io.Reader; |
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:CsvUtil |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2024/3/25 11:02 |
|||
*/ |
|||
public class CsvUtil { |
|||
public static List<Map<String, String>> parseCSV(String filePath, String csvCharSet) { |
|||
List<Map<String, String>> dataList = new ArrayList<>(); |
|||
try (Reader reader = new InputStreamReader(new FileInputStream(filePath), csvCharSet); |
|||
CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT.withFirstRecordAsHeader())) { |
|||
Map<String, Integer> headerMap = csvParser.getHeaderMap(); |
|||
for (CSVRecord csvRecord : csvParser) { |
|||
Map<String, String> dataMap = new HashMap<>(); |
|||
for (Map.Entry<String, Integer> entry : headerMap.entrySet()) { |
|||
String key = entry.getKey(); |
|||
int index = entry.getValue(); |
|||
dataMap.put(key, csvRecord.get(index)); |
|||
} |
|||
dataList.add(dataMap); |
|||
} |
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
} |
|||
|
|||
return dataList; |
|||
} |
|||
|
|||
} |
@ -0,0 +1,60 @@ |
|||
package com.bfd.crawl.ppthandler.util; |
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
import com.alibaba.fastjson.JSONObject; |
|||
import com.alibaba.fastjson.JSONPath; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import java.util.Map; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:DataUtil |
|||
* @version:1.0 |
|||
* @description: 获取dataValue的值 |
|||
* @Date:2023/11/1 9:54 |
|||
*/ |
|||
@Slf4j |
|||
public class DataUtil { |
|||
/** |
|||
* @param key 传入的key |
|||
* @param dataMap 数据map |
|||
* @return 根据传入的参数进行判断解析,返回正确的dataValue |
|||
*/ |
|||
public static Object getValue(String key, Map dataMap) { |
|||
try { |
|||
//公式为空直接就返回 |
|||
if (!StringUtil.hasValue(key)) { |
|||
return ""; |
|||
} |
|||
Object dataValue; |
|||
String isJson = "#json#"; |
|||
if (key.contains(isJson)) { |
|||
//进行第一次拆分,获取#json#前面的部分 |
|||
String[] keySplit = key.split(isJson); |
|||
String firstDataKey = keySplit[0]; |
|||
String[] firstDataKeySplit = firstDataKey.split(":"); |
|||
//取出前半部分对应的JSON数据并转换为JSONObject |
|||
String dataJson = (String) dataMap.get(firstDataKeySplit[0]); |
|||
JSONObject dataJsonObject = JSON.parseObject(dataJson); |
|||
//根据key的后半部分取出对应JSONObject中的值 |
|||
String firstDataKeyJson = (String) JSONPath.eval(dataJsonObject, firstDataKeySplit[1]); |
|||
String secDataKey = keySplit[1]; |
|||
JSONObject firstDataJsonObject = JSON.parseObject(firstDataKeyJson); |
|||
dataValue = JSONPath.eval(firstDataJsonObject, secDataKey); |
|||
return dataValue; |
|||
} |
|||
String[] keySplit = key.split(":"); |
|||
String jsonPath = keySplit[1]; |
|||
String dataJson = (String) dataMap.get(keySplit[0]); |
|||
JSONObject dataJsonObject = JSON.parseObject(dataJson); |
|||
dataValue = JSONPath.eval(dataJsonObject, jsonPath); |
|||
return dataValue; |
|||
} catch (Exception e) { |
|||
// TODO: handle exception |
|||
log.error("jsonpath公式取值异常,", e); |
|||
return null; |
|||
} |
|||
|
|||
} |
|||
} |
@ -0,0 +1,182 @@ |
|||
package com.bfd.crawl.ppthandler.util; |
|||
|
|||
import org.apache.poi.ss.usermodel.*; |
|||
import org.apache.poi.xssf.usermodel.XSSFWorkbook; |
|||
|
|||
import java.io.File; |
|||
import java.io.FileInputStream; |
|||
import java.io.FileOutputStream; |
|||
import java.io.IOException; |
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.nio.file.Paths; |
|||
import java.nio.file.StandardCopyOption; |
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
|
|||
/** |
|||
* @author jian.mao |
|||
* @date 2023年4月7日 |
|||
* @description excel解析工具类 |
|||
*/ |
|||
public class ExcelUtils { |
|||
/** |
|||
* excel解析 |
|||
* |
|||
* @param excel |
|||
* @return |
|||
*/ |
|||
public static Map<String, Object> parse(File excel) { |
|||
Map<String, Object> excelMap = new HashMap<String, Object>(16); |
|||
try { |
|||
FileInputStream file = new FileInputStream(excel); |
|||
// 使用工厂模式创建工作簿对象 |
|||
Workbook workbook = WorkbookFactory.create(file); |
|||
// 获取工作簿中工作表的数量 |
|||
int numberOfSheets = workbook.getNumberOfSheets(); |
|||
DataFormatter dataFormatter = new DataFormatter(); |
|||
// 遍历所有工作表 |
|||
for (int i = 0; i < numberOfSheets; i++) { |
|||
Sheet sheet = workbook.getSheetAt(i); |
|||
String key = sheet.getSheetName(); |
|||
//行码 |
|||
int rowNum = 0; |
|||
List<Map<String, String>> data = new ArrayList<Map<String, String>>(); |
|||
Map<Integer, String> titleHead = new HashMap<Integer, String>(16); |
|||
// 遍历所有行 |
|||
for (Row row : sheet) { |
|||
//单元格码 |
|||
int cellNum = 0; |
|||
//行内容存储 |
|||
Map<String, String> rowMap = new HashMap<String, String>(16); |
|||
// 遍历所有单元格 |
|||
if (rowNum == 0) { |
|||
for (Cell cell : row) { |
|||
String cellValue = dataFormatter.formatCellValue(cell); |
|||
titleHead.put(cellNum, cellValue); |
|||
cellNum++; |
|||
} |
|||
} else { |
|||
for (int j = 0; j < titleHead.size(); j++) { |
|||
String cellValue = dataFormatter.formatCellValue(row.getCell(j)); |
|||
rowMap.put(titleHead.get(cellNum), cellValue); |
|||
cellNum++; |
|||
} |
|||
} |
|||
|
|||
if (rowNum > 0) { |
|||
data.add(rowMap); |
|||
} |
|||
rowNum++; |
|||
} |
|||
excelMap.put(key, data); |
|||
} |
|||
// 关闭文件输入流和工作簿对象 |
|||
file.close(); |
|||
workbook.close(); |
|||
} catch (IOException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
return excelMap; |
|||
} |
|||
|
|||
/** |
|||
* 将List<Map<String, String>>写入Excel文件中 |
|||
* |
|||
* @param data 要写入Excel的数据,每个Map代表一行数据,Map的key为列名,value为单元格数据 |
|||
* @param excelFilePath Excel文件路径,包含文件名和扩展名 |
|||
* @param sheetName 工作表名称 |
|||
* @throws IOException 如果写入Excel文件时发生IO异常,则抛出该异常 |
|||
*/ |
|||
public static void write(List<Map<String, String>> data, String excelFilePath, String sheetName) throws IOException { |
|||
// 创建一个新的工作簿对象 |
|||
Workbook workbook = new XSSFWorkbook(); |
|||
// 创建一个新的工作表 |
|||
Sheet sheet = workbook.createSheet(sheetName); |
|||
// 行码 |
|||
int rowNum = 0; |
|||
// 写入列头 |
|||
Row headerRow = sheet.createRow(rowNum++); |
|||
int colNum = 0; |
|||
for (String key : data.get(0).keySet()) { |
|||
Cell cell = headerRow.createCell(colNum++); |
|||
cell.setCellValue(key); |
|||
} |
|||
// 写入数据 |
|||
for (Map<String, String> rowMap : data) { |
|||
Row row = sheet.createRow(rowNum++); |
|||
colNum = 0; |
|||
for (String key : rowMap.keySet()) { |
|||
Cell cell = row.createCell(colNum++); |
|||
try { |
|||
String s = rowMap.get(key); |
|||
if (s.length() > 30000) { |
|||
|
|||
cell.setCellValue(s.substring(0, 25000)); |
|||
} else { |
|||
cell.setCellValue(s); |
|||
} |
|||
} catch (Exception e) { |
|||
System.out.println(key); |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
} |
|||
// 将数据写入文件 |
|||
FileOutputStream outputStream = new FileOutputStream(excelFilePath); |
|||
workbook.write(outputStream); |
|||
workbook.close(); |
|||
outputStream.close(); |
|||
} |
|||
|
|||
// public static void copyFile(String sourceFloder, String targetFileName) { |
|||
// File sourceFile = new File(sourceFloder); |
|||
// byte[] buffer = new byte[(int) sourceFile.length()]; |
|||
// try (InputStream inputStream = new FileInputStream(sourceFile)) { |
|||
// inputStream.read(buffer); |
|||
// } catch (IOException e) { |
|||
// e.printStackTrace(); |
|||
// return; |
|||
// } |
|||
// // 写入目标文件 |
|||
// File targetFile = new File(targetFileName); |
|||
// targetFile.mkdirs(); |
|||
// try (OutputStream outputStream = new FileOutputStream(targetFile)) { |
|||
// outputStream.write(buffer); |
|||
// } catch (IOException e) { |
|||
// e.printStackTrace(); |
|||
// return; |
|||
// } |
|||
// } |
|||
|
|||
public static void copyFile(String sourceFilePath) { |
|||
// 源文件和目标文件的路径 |
|||
String targetDrive = "F:"; |
|||
|
|||
try { |
|||
// 获取源文件和目标文件的路径信息 |
|||
Path sourcePath = Paths.get(sourceFilePath); |
|||
Path targetPath = Paths.get(targetDrive + sourcePath.toString().substring(2)); |
|||
|
|||
// 如果目标文件的父目录不存在,则创建该目录 |
|||
if (!targetPath.getParent().toFile().exists()) { |
|||
targetPath.getParent().toFile().mkdirs(); |
|||
} |
|||
|
|||
// 进行文件复制 |
|||
Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); |
|||
|
|||
System.out.println("Copied file: " + sourceFilePath + " -> " + targetPath); |
|||
} catch (IOException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
|
|||
public static void main(String[] args) { |
|||
System.out.println(parse(new File("D:\\fromHanler\\xls\\b51484b213ed8fea61f5b99cbdc1490e.xlsx"))); |
|||
} |
|||
|
|||
} |
@ -0,0 +1,117 @@ |
|||
package com.bfd.crawl.ppthandler.util; |
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
import okhttp3.*; |
|||
|
|||
import java.io.File; |
|||
import java.io.FileOutputStream; |
|||
import java.io.IOException; |
|||
import java.io.InputStream; |
|||
import java.util.HashMap; |
|||
import java.util.Map; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:FileDownloader |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/8/1 16:31 |
|||
*/ |
|||
public class FileDownloader { |
|||
private static OkHttpClient okHttpClient; |
|||
|
|||
private static OkHttpClient getOkHttpClient() { |
|||
if (okHttpClient == null) { |
|||
okHttpClient = new OkHttpClient(); |
|||
} |
|||
return okHttpClient; |
|||
} |
|||
|
|||
public static void downloadFile(String url, File destination) throws IOException { |
|||
OkHttpClient client = getOkHttpClient(); |
|||
Request request = new Request.Builder() |
|||
.url(url) |
|||
.build(); |
|||
|
|||
try (Response response = client.newCall(request).execute()) { |
|||
if (!response.isSuccessful()) { |
|||
throw new IOException("Failed to download file: " + response); |
|||
} |
|||
|
|||
ResponseBody body = response.body(); |
|||
if (body == null) { |
|||
throw new IOException("Response body is null"); |
|||
} |
|||
if (!destination.getParentFile().exists()) { |
|||
|
|||
destination.getParentFile().mkdirs(); |
|||
} |
|||
try (InputStream inputStream = body.byteStream(); |
|||
FileOutputStream outputStream = new FileOutputStream(destination)) { |
|||
byte[] buffer = new byte[8192]; |
|||
int bytesRead; |
|||
while ((bytesRead = inputStream.read(buffer)) != -1) { |
|||
outputStream.write(buffer, 0, bytesRead); |
|||
} |
|||
outputStream.flush(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public static Map<String, String> uploadFile(String url, String filePath) throws Exception { |
|||
File file = new File(filePath); |
|||
|
|||
Map returnMap = new HashMap(32); |
|||
OkHttpClient client = getOkHttpClient(); |
|||
// 设置文件上传的媒体类型 |
|||
MediaType mediaType = MediaType.parse("application/octet-stream"); |
|||
// 创建请求体,将文件添加到请求体中 |
|||
RequestBody requestBody = RequestBody.create(mediaType, file); |
|||
|
|||
// 创建多部分请求体,用于上传文件 |
|||
MultipartBody multipartBody = new MultipartBody.Builder() |
|||
.setType(MultipartBody.FORM) |
|||
.addFormDataPart("file", file.getName(), requestBody) |
|||
.build(); |
|||
// 创建上传文件的请求 |
|||
Request request = new Request.Builder() |
|||
.url(url) |
|||
.post(multipartBody) |
|||
.build(); |
|||
|
|||
try (Response response = client.newCall(request).execute()) { |
|||
if (!response.isSuccessful()) { |
|||
throw new IOException("Failed to upload file: " + response); |
|||
} |
|||
String html = response.body().string(); |
|||
|
|||
try { |
|||
Map parse = (Map) JSON.parse(html); |
|||
Map data = (Map) parse.get("data"); |
|||
String domain = (String) data.get("domain"); |
|||
String src = (String) data.get("src"); |
|||
String fileUrl = domain.concat(src); |
|||
returnMap.put("fileUrl", fileUrl); |
|||
} catch (Exception e) { |
|||
returnMap.put("fileUrl", html); |
|||
} |
|||
// 处理上传成功的响应 |
|||
System.out.println("File uploaded successfully!"); |
|||
} |
|||
return returnMap; |
|||
} |
|||
|
|||
public static void deleteFile(String url, String md5) throws Exception { |
|||
OkHttpClient client = getOkHttpClient(); |
|||
url = url.concat("delete?md5=").concat(md5); |
|||
Request request = new Request.Builder() |
|||
.url(url) |
|||
.get() |
|||
.build(); |
|||
try (Response response = client.newCall(request).execute()) { |
|||
if (!response.isSuccessful()) { |
|||
throw new IOException("Failed to upload file: " + response); |
|||
} |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,42 @@ |
|||
package com.bfd.crawl.ppthandler.util; |
|||
|
|||
import java.io.File; |
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:FileUtil |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/8/2 10:57 |
|||
*/ |
|||
public class FileUtil { |
|||
|
|||
public static void main(String[] args) { |
|||
System.out.println(traverseAndReturnFilePath("D:\\\\ocr\\\\305ce27d7a05770456fdc09d0b3044f7\\\\")); |
|||
} |
|||
|
|||
public static List<String> traverseAndReturnFilePath(String folderPath) { |
|||
List<String> fileList = new ArrayList<>(); |
|||
File folder = new File(folderPath); |
|||
// 检查文件夹是否存在并且是一个文件夹 |
|||
if (folder.exists() && folder.isDirectory()) { |
|||
// 获取文件夹中的所有文件和子文件夹 |
|||
File[] files = folder.listFiles(); |
|||
if (files != null) { |
|||
for (File file : files) { |
|||
if (file.isFile()) { |
|||
// 如果是文件,则输出全路径 |
|||
fileList.add(file.getAbsolutePath()); |
|||
} else if (file.isDirectory()) { |
|||
|
|||
} |
|||
} |
|||
} |
|||
} else { |
|||
System.out.println("指定的路径不是一个文件夹或文件夹不存在。"); |
|||
} |
|||
return fileList; |
|||
} |
|||
} |
@ -0,0 +1,59 @@ |
|||
package com.bfd.crawl.ppthandler.util; |
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
import okhttp3.*; |
|||
|
|||
import java.util.Map; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:ocrUtil |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/8/1 16:38 |
|||
*/ |
|||
public class OcrUtil { |
|||
private static OkHttpClient okHttpClient; |
|||
|
|||
private static OkHttpClient getOkHttpClient() { |
|||
if (okHttpClient == null) { |
|||
okHttpClient = new OkHttpClient(); |
|||
} |
|||
return okHttpClient; |
|||
} |
|||
|
|||
public static String doOcr(String url,String ocrApi) { |
|||
String text = ""; |
|||
int reTryTimes = 3; |
|||
for (int i = 0; i < reTryTimes; i++) { |
|||
int okCode = 200; |
|||
OkHttpClient client = getOkHttpClient(); |
|||
OkHttpClient.Builder builder = client.newBuilder().writeTimeout(600, TimeUnit.SECONDS).connectTimeout(600, TimeUnit.SECONDS).readTimeout(600, TimeUnit.SECONDS); |
|||
client = builder.build(); |
|||
MediaType mediaType = MediaType.parse("application/json"); |
|||
RequestBody body = RequestBody.create(mediaType, "{\"id\":\"\",\"url\":\"" + url + "\"}"); |
|||
Request request = new Request.Builder() |
|||
.url(ocrApi) |
|||
.method("POST", body) |
|||
.addHeader("Content-Type", "application/json") |
|||
.build(); |
|||
try { |
|||
Response response = client.newCall(request).execute(); |
|||
String html = response.body().string(); |
|||
Map dataMap = (Map) JSON.parse(html); |
|||
int code = (int) dataMap.get("code"); |
|||
if (code == okCode) { |
|||
text = (String) dataMap.get("text"); |
|||
} |
|||
if (StringUtil.hasValue(text)) { |
|||
break; |
|||
} |
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
|
|||
return text; |
|||
} |
|||
} |
@ -0,0 +1,23 @@ |
|||
package com.bfd.crawl.ppthandler.util; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import java.util.Locale; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:OsUtil |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/4/23 9:40 |
|||
*/ |
|||
@Slf4j |
|||
public class OsUtil { |
|||
public static Boolean isWindows() { |
|||
String windows = "windows"; |
|||
String osName = System.getProperty("os.name").toLowerCase(Locale.ROOT); |
|||
log.info("osName = " + osName); |
|||
return osName.contains(windows); |
|||
} |
|||
|
|||
} |
@ -0,0 +1,124 @@ |
|||
package com.bfd.crawl.ppthandler.util; |
|||
|
|||
import org.apache.poi.hslf.usermodel.HSLFShape; |
|||
import org.apache.poi.hslf.usermodel.HSLFSlide; |
|||
import org.apache.poi.hslf.usermodel.HSLFSlideShow; |
|||
import org.apache.poi.hslf.usermodel.HSLFTextShape; |
|||
import org.apache.poi.xslf.usermodel.*; |
|||
|
|||
import java.io.FileInputStream; |
|||
import java.io.IOException; |
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:PptUtil |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2024/3/25 16:14 |
|||
*/ |
|||
public class PptUtil { |
|||
|
|||
|
|||
public static String parsepptx(String filePath) { |
|||
StringBuilder dataStringsb = new StringBuilder(); |
|||
XMLSlideShow ppt = null; |
|||
try { |
|||
// PPT类 |
|||
ppt = new XMLSlideShow(new FileInputStream(filePath)); |
|||
// 获取PPT中的所有幻灯片 |
|||
// List<XSLFSlide> slides = ppt.getSlides(); |
|||
// 遍历幻灯片 |
|||
for (int i = 0; i < ppt.getSlides().size(); i++) { |
|||
|
|||
//拿到第i页的PPT |
|||
XSLFSlide slides = ppt.getSlides().get(i); |
|||
System.out.println("第" + (i + 1) + "页"); |
|||
//注释的for循环是获取所以PPT的内容 |
|||
// for (XSLFSlide slide : slides) { |
|||
// 获取幻灯片中的所有图形 |
|||
List<XSLFShape> shapes = slides.getShapes(); |
|||
// 遍历PPT的图形 |
|||
for (XSLFShape shape : shapes) { |
|||
// 判断该图形类是否是文本框类 |
|||
if (shape instanceof XSLFTextShape) { |
|||
// 将图像类强制装换成文本框类 |
|||
XSLFTextShape ts = (XSLFTextShape) shape; |
|||
// 获取文本框内的文字 |
|||
String str = ts.getText(); |
|||
dataStringsb.append(str); |
|||
} |
|||
// 判断该图形类是否是表格类 |
|||
if (shape instanceof XSLFTable) { |
|||
// 将图像类强制装换成表格类 |
|||
XSLFTable table = (XSLFTable) shape; |
|||
// 获取表格中的所有行 |
|||
List<XSLFTableRow> rows = table.getRows(); |
|||
for (XSLFTableRow tr : rows) { |
|||
// 获取行中的所有单元格 |
|||
List<XSLFTableCell> cells = tr.getCells(); |
|||
for (XSLFTableCell tc : cells) { |
|||
// 获取单元格内的文字 |
|||
String str = tc.getText(); |
|||
dataStringsb.append(str); |
|||
} |
|||
} |
|||
} |
|||
// 判断该图形类是否是图片框类chr |
|||
if (shape instanceof XSLFPictureShape) { |
|||
// 将图像类强制装换成图片框类 |
|||
XSLFPictureShape ps = (XSLFPictureShape) shape; |
|||
// 获取图片的字节码数据(可以利用输出流将该图片保存到硬盘里) |
|||
byte[] pictureData = ps.getPictureData().getData(); |
|||
// System.out.println("图片信息:" + pictureData); |
|||
} |
|||
} |
|||
|
|||
} |
|||
// } |
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
} finally { |
|||
if (ppt != null) { |
|||
|
|||
try { |
|||
// 保存完之后要对PPT进行关闭操作 |
|||
ppt.close(); |
|||
} catch (IOException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
|
|||
} |
|||
|
|||
} |
|||
return dataStringsb.toString(); |
|||
} |
|||
|
|||
public static String parsePpt(String filePath) { |
|||
StringBuilder dataStringsb = new StringBuilder(); |
|||
HSLFSlideShow ppt = null; |
|||
try { |
|||
ppt = new HSLFSlideShow(new FileInputStream(filePath)); |
|||
for (HSLFSlide slide : ppt.getSlides()) { |
|||
for (HSLFShape shape : slide.getShapes()) { |
|||
if (shape instanceof HSLFTextShape) { |
|||
HSLFTextShape textShape = (HSLFTextShape) shape; |
|||
dataStringsb.append(textShape.getText()).append("\n"); |
|||
} |
|||
} |
|||
} |
|||
} catch (Exception e) { |
|||
e.printStackTrace(); |
|||
} finally { |
|||
if (ppt != null) { |
|||
try { |
|||
ppt.close(); |
|||
} catch (IOException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
} |
|||
return dataStringsb.toString(); |
|||
} |
|||
} |
@ -0,0 +1,19 @@ |
|||
package com.bfd.crawl.ppthandler.util; |
|||
|
|||
import java.util.concurrent.LinkedBlockingDeque; |
|||
|
|||
/** |
|||
* @author:jinming |
|||
* @className:QueueUtil |
|||
* @version:1.0 |
|||
* @description: |
|||
* @Date:2023/7/13 15:00 |
|||
*/ |
|||
public class QueueUtil { |
|||
|
|||
|
|||
public static LinkedBlockingDeque<String> taskQueue = new LinkedBlockingDeque<String>(); |
|||
|
|||
public static LinkedBlockingDeque<String> sendQueue = new LinkedBlockingDeque<String>(); |
|||
|
|||
} |
@ -0,0 +1,94 @@ |
|||
package com.bfd.crawl.ppthandler.util; |
|||
|
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
|
|||
import java.security.MessageDigest; |
|||
import java.util.HashSet; |
|||
import java.util.Set; |
|||
import java.util.regex.Matcher; |
|||
import java.util.regex.Pattern; |
|||
|
|||
/** |
|||
* @author jinming |
|||
* @version 1.0 |
|||
* @className StringUtile |
|||
* @Date 2022/1/21 11:46 |
|||
*/ |
|||
@Slf4j |
|||
public class StringUtil { |
|||
public static boolean hasValue(String str) { |
|||
return str != null && !"".equals(str.trim()); |
|||
} |
|||
|
|||
public static String getRegexGroup(String regex, String str, int id) { |
|||
String resultStr = ""; |
|||
if (hasValue(str)) { |
|||
Pattern p = Pattern.compile(regex); |
|||
Matcher m = p.matcher(str); |
|||
if (m.find()) { |
|||
resultStr = m.group(id); |
|||
} |
|||
} |
|||
|
|||
if ("".equals(resultStr)) { |
|||
} |
|||
|
|||
return resultStr; |
|||
} |
|||
|
|||
public static Set<String> getEmailAddress(String message) { |
|||
Set<String> emailList = new HashSet<>(); |
|||
Pattern pattern = Pattern.compile("\\w+\\.?\\w+\\@\\w+\\.\\w+"); |
|||
Matcher m = pattern.matcher(message); |
|||
while (m.find()) { |
|||
emailList.add(m.group(0)); |
|||
} |
|||
return emailList; |
|||
} |
|||
public static String getMd5(String string) { |
|||
try { |
|||
MessageDigest md5 = MessageDigest.getInstance("MD5"); |
|||
byte[] bs = md5.digest(string.getBytes("UTF-8")); |
|||
StringBuilder sb = new StringBuilder(40); |
|||
for (byte x : bs) { |
|||
if ((x & 0xff) >> 4 == 0) { |
|||
sb.append("0").append(Integer.toHexString(x & 0xff)); |
|||
} else { |
|||
sb.append(Integer.toHexString(x & 0xff)); |
|||
} |
|||
} |
|||
return sb.toString(); |
|||
} catch (Exception e) { |
|||
//LOG.error("获取md5异常", e); |
|||
return "nceaform" + System.currentTimeMillis(); |
|||
} |
|||
} |
|||
|
|||
public static String removeAllHtmlTags(String str) { |
|||
return hasValue(str) ? str.replaceAll("<[^<>]+?>", "") : ""; |
|||
} |
|||
|
|||
public static String getRegexGroup(Pattern regex, String str, int id) { |
|||
String resultStr = ""; |
|||
if (hasValue(str)) { |
|||
Matcher m = regex.matcher(str); |
|||
if (m.find()) { |
|||
resultStr = m.group(id); |
|||
} |
|||
} |
|||
|
|||
if ("".equals(resultStr)) { |
|||
log.error(regex + " parser error!"); |
|||
} |
|||
|
|||
return resultStr; |
|||
} |
|||
|
|||
public static String getStrByPattern(String str, String regex) { |
|||
Pattern pattern = Pattern.compile(regex); |
|||
Matcher m = pattern.matcher(str); |
|||
return m.find() ? m.group(0) : ""; |
|||
} |
|||
|
|||
} |
@ -0,0 +1,47 @@ |
|||
server: |
|||
port: 7082 |
|||
spring: |
|||
application: |
|||
name: 幻灯片处理 |
|||
boot: |
|||
admin: |
|||
client: |
|||
health: |
|||
timeout: 10s |
|||
url: http://172.16.12.55:8001 |
|||
instance: |
|||
service-base-url: http://172.16.12.55:7088 |
|||
kafka: |
|||
bootstrap-servers: 172.26.28.30:9092 |
|||
producer: |
|||
retries: 3 |
|||
acks: all |
|||
batch-size: 4096 |
|||
buffer-memory: 102476800 |
|||
key-serializer: org.apache.kafka.common.serialization.StringSerializer |
|||
value-serializer: org.apache.kafka.common.serialization.StringSerializer |
|||
|
|||
|
|||
logging: |
|||
file: |
|||
path: ./logs |
|||
|
|||
management: |
|||
endpoints: |
|||
web: |
|||
exposure: |
|||
include: "*" |
|||
endpoint: |
|||
health: |
|||
show-details: always |
|||
|
|||
send: |
|||
topic: analyze0912 |
|||
|
|||
file: |
|||
path: D:\\fromHanler\\ |
|||
|
|||
thread: |
|||
handler: 1 |
|||
send: 1 |
|||
|
@ -0,0 +1,36 @@ |
|||
<configuration> |
|||
<!-- 属性文件:在properties文件中找到对应的配置项 --> |
|||
<springProperty scope="context" name="logging.file.path" source="logging.file.path"/> |
|||
<springProperty scope="context" name="logging.level" source="logging.level"/> |
|||
<!-- 默认的控制台日志输出,一般生产环境都是后台启动,这个没太大作用 --> |
|||
<appender name="STDOUT" |
|||
class="ch.qos.logback.core.ConsoleAppender"> |
|||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> |
|||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</pattern> |
|||
</encoder> |
|||
</appender> |
|||
|
|||
<appender name="GLMAPPER-LOGGERONE" |
|||
class="ch.qos.logback.core.rolling.RollingFileAppender"> |
|||
<append>true</append> |
|||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
|||
<level>${logging.level}</level> |
|||
</filter> |
|||
<file> |
|||
${logging.file.path}/formHandler.log |
|||
</file> |
|||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> |
|||
<FileNamePattern>${logging.file.path}/formHandler.log.%d{yyyy-MM-dd}</FileNamePattern> |
|||
<MaxHistory>3</MaxHistory> |
|||
</rollingPolicy> |
|||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> |
|||
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</pattern> |
|||
<charset>UTF-8</charset> |
|||
</encoder> |
|||
</appender> |
|||
|
|||
<root level="info"> |
|||
<appender-ref ref="GLMAPPER-LOGGERONE"/> |
|||
<appender-ref ref="STDOUT"/> |
|||
</root> |
|||
</configuration> |
Write
Preview
Loading…
Cancel
Save
Reference in new issue