Browse Source

ppt解析应用

master
55007 6 months ago
commit
cb8c11dfdd
  1. 33
      .gitignore
  2. 1
      README.md
  3. 138
      pom.xml
  4. 13
      src/main/java/com/bfd/crawl/ppthandler/PptHanlerApplication.java
  5. 60
      src/main/java/com/bfd/crawl/ppthandler/bean/ResponsePo.java
  6. 48
      src/main/java/com/bfd/crawl/ppthandler/config/AsyncThreadConfiguration.java
  7. 27
      src/main/java/com/bfd/crawl/ppthandler/config/Constant.java
  8. 46
      src/main/java/com/bfd/crawl/ppthandler/controller/PptHandlerController.java
  9. 32
      src/main/java/com/bfd/crawl/ppthandler/enums/ResponseCode.java
  10. 121
      src/main/java/com/bfd/crawl/ppthandler/service/HandlerService.java
  11. 54
      src/main/java/com/bfd/crawl/ppthandler/service/SendService.java
  12. 63
      src/main/java/com/bfd/crawl/ppthandler/service/StartServcie.java
  13. 44
      src/main/java/com/bfd/crawl/ppthandler/util/CsvUtil.java
  14. 60
      src/main/java/com/bfd/crawl/ppthandler/util/DataUtil.java
  15. 182
      src/main/java/com/bfd/crawl/ppthandler/util/ExcelUtils.java
  16. 117
      src/main/java/com/bfd/crawl/ppthandler/util/FileDownloader.java
  17. 42
      src/main/java/com/bfd/crawl/ppthandler/util/FileUtil.java
  18. 59
      src/main/java/com/bfd/crawl/ppthandler/util/OcrUtil.java
  19. 23
      src/main/java/com/bfd/crawl/ppthandler/util/OsUtil.java
  20. 124
      src/main/java/com/bfd/crawl/ppthandler/util/PptUtil.java
  21. 19
      src/main/java/com/bfd/crawl/ppthandler/util/QueueUtil.java
  22. 94
      src/main/java/com/bfd/crawl/ppthandler/util/StringUtil.java
  23. 47
      src/main/resources/application.yml
  24. 36
      src/main/resources/logback-spring.xml

33
.gitignore

@ -0,0 +1,33 @@
HELP.md
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/

1
README.md

@ -0,0 +1 @@
ppt 解析应用

138
pom.xml

@ -0,0 +1,138 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.4.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.bfd.crawl</groupId>
<artifactId>pptHandler</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>pptHandler</name>
<description>pptHandler</description>
<properties>
<java.version>8</java.version>
</properties>
<dependencies>
<!-- https://mvnrepository.com/artifact/de.codecentric/spring-boot-admin-starter-client -->
<dependency>
<groupId>de.codecentric</groupId>
<artifactId>spring-boot-admin-client</artifactId>
<version>2.2.4</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.10.0</version>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>2.0.28</version>
</dependency>
<!-- Apache POI -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-scratchpad</artifactId>
<version>5.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>5.2.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.17.1</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>5.2.0</version>
</dependency>
<!--JSON-->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>2.0.17</version>
</dependency>
<!--OKHTTP-->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>3.9.1</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.8</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.3.1</version> <!--根据您正在使用的Kafka版本选择合适的版本号-->
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<excludes>
<exclude>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>

13
src/main/java/com/bfd/crawl/ppthandler/PptHanlerApplication.java

@ -0,0 +1,13 @@
package com.bfd.crawl.ppthandler;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class PptHanlerApplication {
public static void main(String[] args) {
SpringApplication.run(PptHanlerApplication.class, args);
}
}

60
src/main/java/com/bfd/crawl/ppthandler/bean/ResponsePo.java

@ -0,0 +1,60 @@
package com.bfd.crawl.ppthandler.bean;
import com.bfd.crawl.ppthandler.enums.ResponseCode;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @author:jinming
* @className:ResponsePo
* @version:1.0
* @description:
* @Date:2023/4/3 17:23
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ResponsePo {
/**
* 响应码
*/
private int code;
/**
* 正常放 返回数据 的JSON串
*/
private Object data;
/**
* 提示消息
*/
private String message;
public static ResponsePo success() {
return setStatus(ResponseCode.SUCCESS.getCode(), ResponseCode.SUCCESS.getMessage());
}
public static ResponsePo error() {
return setStatus(ResponseCode.FAILURE.getCode(), ResponseCode.FAILURE.getMessage());
}
public static ResponsePo setStatus(int code, String message) {
ResponsePo resultBean = new ResponsePo();
resultBean.code = code;
resultBean.message = message;
return resultBean;
}
public ResponsePo(int code, String message) {
this.code = code;
this.message = message;
this.data = data;
}
public ResponsePo(ResponseCode responseCode){
this.code = responseCode.getCode();
this.message = responseCode.getMessage();
this.data = data;
}
}

48
src/main/java/com/bfd/crawl/ppthandler/config/AsyncThreadConfiguration.java

@ -0,0 +1,48 @@
package com.bfd.crawl.ppthandler.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.Executor;
/**
* @author jinming
* @version 1.0
* @className AsyncThreadConfiguration
* @Date 2022/2/17 18:37
*/
@Configuration
@EnableAsync
public class AsyncThreadConfiguration {
@Bean
public Executor asyncExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 核心线程数
executor.setCorePoolSize(500);
// 并发线程的数量限制为2
executor.setMaxPoolSize(500);
// 线程队列
executor.setQueueCapacity(500);
executor.setThreadNamePrefix("handlerData-");
executor.initialize();
executor.setWaitForTasksToCompleteOnShutdown(true);
return executor;
}
@Bean
public Executor sendExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 核心线程数
executor.setCorePoolSize(500);
// 并发线程的数量限制为2
executor.setMaxPoolSize(500);
// 线程队列
executor.setQueueCapacity(500);
executor.setThreadNamePrefix("sendData-");
executor.initialize();
executor.setWaitForTasksToCompleteOnShutdown(true);
return executor;
}
}

27
src/main/java/com/bfd/crawl/ppthandler/config/Constant.java

@ -0,0 +1,27 @@
package com.bfd.crawl.ppthandler.config;
/**
* @author:jinming
* @className:Constant
* @version:1.0
* @description:
* @Date:2023/8/16 15:26
*/
public class Constant {
/**
*
*/
public final static String IS_XLS = "xls";
/**
*
*/
public final static String IS_CSV = "csv";
/**
*
*/
public final static String ALL = "*";
}

46
src/main/java/com/bfd/crawl/ppthandler/controller/PptHandlerController.java

@ -0,0 +1,46 @@
package com.bfd.crawl.ppthandler.controller;
import com.alibaba.fastjson.JSON;
import com.bfd.crawl.ppthandler.bean.ResponsePo;
import com.bfd.crawl.ppthandler.enums.ResponseCode;
import com.bfd.crawl.ppthandler.util.QueueUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.Map;
/**
* @author:jinming
* @className:DataFilterController
* @version:1.0
* @description:
* @Date:2023/7/26 11:21
*/
@RestController
@RequestMapping("/handlerdata")
@Slf4j
public class PptHandlerController {
@PostMapping("/ppthandler")
public ResponsePo documentFeedback(@RequestBody String dataJson) {
ResponsePo responsePo = ResponsePo.success();
try {
Map parse = (Map) JSON.parse(dataJson);
} catch (Exception e) {
e.printStackTrace();
log.error("请求格式发生异常" + e.getMessage());
responsePo.setCode(ResponseCode.FAILURE.getCode());
responsePo.setMessage(ResponseCode.FAILURE.getMessage());
return responsePo;
}
log.info("新增任务:" + dataJson);
QueueUtil.taskQueue.add(dataJson);
return responsePo;
}
}

32
src/main/java/com/bfd/crawl/ppthandler/enums/ResponseCode.java

@ -0,0 +1,32 @@
package com.bfd.crawl.ppthandler.enums;
/**
* @author:jinming
* @className:ResponseCodeEnum
* @version:1.0
* @description:响应结果码枚举类
* @Date:2023/2/28 11:40
*/
public enum ResponseCode {
//返回结果码枚举类
SUCCESS(200, "操作成功"),
FAILURE(400, "参数错误"),
INTERNAL_SERVER_ERROR(500, "服务器内部错误"),
TYPE_NOT_SUPPORT(601,"文件类型不支持");
private int code;
private String message;
ResponseCode(int code, String message) {
this.code = code;
this.message = message;
}
public int getCode() {
return code;
}
public String getMessage() {
return message;
}
}

121
src/main/java/com/bfd/crawl/ppthandler/service/HandlerService.java

@ -0,0 +1,121 @@
package com.bfd.crawl.ppthandler.service;
import com.alibaba.fastjson.JSON;
import com.bfd.crawl.ppthandler.config.Constant;
import com.bfd.crawl.ppthandler.util.*;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* @author:jinming
* @className:HandlerService
* @version:1.0
* @description:
* @Date:2023/8/1 16:05
*/
@Service
@Slf4j
public class HandlerService {
@Value("${file.path}")
private String downloadFilePath;
@Async("asyncExecutor")
void run() {
while (true) {
try {
if (QueueUtil.taskQueue.size() > 0) {
String dataJson = null;
try {
dataJson = QueueUtil.taskQueue.take();
} catch (InterruptedException e) {
e.printStackTrace();
continue;
}
String errorMessage = "";
Map parse = (Map) JSON.parse(dataJson);
Map dataMap = (Map) parse.get("data");
int id = (int) parse.get("id");
Map admin = (Map) parse.get("input");
String key = (String) admin.get("fileUrl");
String fileUrl = (String) DataUtil.getValue(key, dataMap);
boolean windows = OsUtil.isWindows();
Map output = (Map) parse.get("output");
// List<String> sheetName = (List<String>) admin.get("sheetName");
//fieldType自定义输出字段 0 关闭1-开启如果开启则拼接form到output里如果关闭则取默认的output拼接
String fileType = StringUtil.getStrByPattern(fileUrl, "\\.[^.\\\\/:*?\"<>|\\r\\n]+$");
log.info("任务:" + id + "的文件类型为" + fileType);
String fileName = StringUtil.getMd5(fileUrl);
//定义ppt的下载路径
String pptDir = null;
try {
pptDir = downloadFilePath.concat(windows ? "\\ppt\\" : "/ppt/").concat(fileName).concat(fileType);
FileDownloader.downloadFile(fileUrl, new File(pptDir));
} catch (IOException e) {
e.printStackTrace();
errorMessage = "文件下载失败";
}
String pptParse = null;
String pptx = "pptx";
try {
if (fileType.contains(pptx)) {
pptParse = PptUtil.parsepptx(pptDir);
} else {
pptParse = PptUtil.parsePpt(pptDir);
}
} catch (Exception e) {
e.printStackTrace();
errorMessage = "文件解析失败";
}
if (!StringUtil.hasValue(errorMessage)) {
Map result = new HashMap(32);
Map resultsMap = new HashMap(32);
resultsMap.put("result", pptParse);
resultsMap.put("isLast", 1);
result.put("status", 1);
result.put("message", "成功");
result.put("results", JSON.toJSONString(resultsMap));
parse.put("result", result);
String message = JSON.toJSONString(parse);
QueueUtil.sendQueue.put(message);
continue;
}
Map result = new HashMap(32);
Map resultsMap = new HashMap(32);
resultsMap.put("content", errorMessage);
resultsMap.put("isLast", 1);
result.put("results", JSON.toJSONString(resultsMap));
result.put("status", 2);
result.put("message", errorMessage);
parse.put("result", result);
String message = JSON.toJSONString(parse);
QueueUtil.sendQueue.put(message);
} else {
log.info("任务队列为空,休眠3秒");
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
} catch (Throwable e) {
e.printStackTrace();
log.error("工作线程发生异常" + e.getMessage());
}
}
}
}

54
src/main/java/com/bfd/crawl/ppthandler/service/SendService.java

@ -0,0 +1,54 @@
package com.bfd.crawl.ppthandler.service;
import com.alibaba.fastjson.JSON;
import com.bfd.crawl.ppthandler.util.QueueUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.Map;
/**
* @author:jinming
* @className:SendService
* @version:1.0
* @description:
* @Date:2023/7/31 17:53
*/
@Slf4j
@Service
public class SendService {
@Value("${send.topic}")
private String topic;
@Resource
private KafkaTemplate kafkaTemplate;
@Async("sendExecutor")
void sendToKafka() {
while (true) {
if (QueueUtil.sendQueue.size() > 0) {
try {
String message = QueueUtil.sendQueue.take();
Map parse = (Map) JSON.parse(message);
String id = parse.get("id").toString();
log.info("ID:" + id + "\t" + "数据已发出");
kafkaTemplate.send(topic, message);
} catch (Exception e) {
e.printStackTrace();
}
} else {
log.info("任务队列为空,休眠3秒");
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
}

63
src/main/java/com/bfd/crawl/ppthandler/service/StartServcie.java

@ -0,0 +1,63 @@
package com.bfd.crawl.ppthandler.service;
import com.bfd.crawl.ppthandler.util.QueueUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Service;
/**
* @author:jinming
* @className:StartServcie
* @version:1.0
* @description:
* @Date:2023/7/31 17:14
*/
@Service
@Slf4j
public class StartServcie implements ApplicationRunner {
@Value("${thread.handler}")
private int handlerNumber;
@Value("${thread.send}")
private int sendNumber;
@Autowired
private HandlerService handlerService;
@Autowired
private SendService sendService;
@Override
public void run(ApplicationArguments args) throws Exception {
for (int i = 0; i < handlerNumber; i++) {
log.info("处理服务线程" + i + "已启动 ");
handlerService.run();
}
for (int i = 0; i < sendNumber; i++) {
log.info("发送服务线程" + i + "已启动 ");
sendService.sendToKafka();
}
Runnable myRunnable = new Runnable() {
@Override
public void run() {
while (true) {
log.info("任务队列长度为" + QueueUtil.taskQueue.size());
log.info("发送队列长度为" + QueueUtil.sendQueue.size());
try {
Thread.sleep(10000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
};
// 创建一个新的线程并将Runnable对象传递给Thread构造函数
Thread myThread = new Thread(myRunnable);
// 启动线程
myThread.start();
}
}

44
src/main/java/com/bfd/crawl/ppthandler/util/CsvUtil.java

@ -0,0 +1,44 @@
package com.bfd.crawl.ppthandler.util;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author:jinming
* @className:CsvUtil
* @version:1.0
* @description:
* @Date:2024/3/25 11:02
*/
public class CsvUtil {
public static List<Map<String, String>> parseCSV(String filePath, String csvCharSet) {
List<Map<String, String>> dataList = new ArrayList<>();
try (Reader reader = new InputStreamReader(new FileInputStream(filePath), csvCharSet);
CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT.withFirstRecordAsHeader())) {
Map<String, Integer> headerMap = csvParser.getHeaderMap();
for (CSVRecord csvRecord : csvParser) {
Map<String, String> dataMap = new HashMap<>();
for (Map.Entry<String, Integer> entry : headerMap.entrySet()) {
String key = entry.getKey();
int index = entry.getValue();
dataMap.put(key, csvRecord.get(index));
}
dataList.add(dataMap);
}
} catch (Exception e) {
e.printStackTrace();
}
return dataList;
}
}

60
src/main/java/com/bfd/crawl/ppthandler/util/DataUtil.java

@ -0,0 +1,60 @@
package com.bfd.crawl.ppthandler.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONPath;
import lombok.extern.slf4j.Slf4j;
import java.util.Map;
/**
* @author:jinming
* @className:DataUtil
* @version:1.0
* @description: 获取dataValue的值
* @Date:2023/11/1 9:54
*/
@Slf4j
public class DataUtil {
/**
* @param key 传入的key
* @param dataMap 数据map
* @return 根据传入的参数进行判断解析返回正确的dataValue
*/
public static Object getValue(String key, Map dataMap) {
try {
//公式为空直接就返回
if (!StringUtil.hasValue(key)) {
return "";
}
Object dataValue;
String isJson = "#json#";
if (key.contains(isJson)) {
//进行第一次拆分获取#json#前面的部分
String[] keySplit = key.split(isJson);
String firstDataKey = keySplit[0];
String[] firstDataKeySplit = firstDataKey.split(":");
//取出前半部分对应的JSON数据并转换为JSONObject
String dataJson = (String) dataMap.get(firstDataKeySplit[0]);
JSONObject dataJsonObject = JSON.parseObject(dataJson);
//根据key的后半部分取出对应JSONObject中的值
String firstDataKeyJson = (String) JSONPath.eval(dataJsonObject, firstDataKeySplit[1]);
String secDataKey = keySplit[1];
JSONObject firstDataJsonObject = JSON.parseObject(firstDataKeyJson);
dataValue = JSONPath.eval(firstDataJsonObject, secDataKey);
return dataValue;
}
String[] keySplit = key.split(":");
String jsonPath = keySplit[1];
String dataJson = (String) dataMap.get(keySplit[0]);
JSONObject dataJsonObject = JSON.parseObject(dataJson);
dataValue = JSONPath.eval(dataJsonObject, jsonPath);
return dataValue;
} catch (Exception e) {
// TODO: handle exception
log.error("jsonpath公式取值异常,", e);
return null;
}
}
}

182
src/main/java/com/bfd/crawl/ppthandler/util/ExcelUtils.java

@ -0,0 +1,182 @@
package com.bfd.crawl.ppthandler.util;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author jian.mao
* @date 2023年4月7日
* @description excel解析工具类
*/
public class ExcelUtils {
/**
* excel解析
*
* @param excel
* @return
*/
public static Map<String, Object> parse(File excel) {
Map<String, Object> excelMap = new HashMap<String, Object>(16);
try {
FileInputStream file = new FileInputStream(excel);
// 使用工厂模式创建工作簿对象
Workbook workbook = WorkbookFactory.create(file);
// 获取工作簿中工作表的数量
int numberOfSheets = workbook.getNumberOfSheets();
DataFormatter dataFormatter = new DataFormatter();
// 遍历所有工作表
for (int i = 0; i < numberOfSheets; i++) {
Sheet sheet = workbook.getSheetAt(i);
String key = sheet.getSheetName();
//行码
int rowNum = 0;
List<Map<String, String>> data = new ArrayList<Map<String, String>>();
Map<Integer, String> titleHead = new HashMap<Integer, String>(16);
// 遍历所有行
for (Row row : sheet) {
//单元格码
int cellNum = 0;
//行内容存储
Map<String, String> rowMap = new HashMap<String, String>(16);
// 遍历所有单元格
if (rowNum == 0) {
for (Cell cell : row) {
String cellValue = dataFormatter.formatCellValue(cell);
titleHead.put(cellNum, cellValue);
cellNum++;
}
} else {
for (int j = 0; j < titleHead.size(); j++) {
String cellValue = dataFormatter.formatCellValue(row.getCell(j));
rowMap.put(titleHead.get(cellNum), cellValue);
cellNum++;
}
}
if (rowNum > 0) {
data.add(rowMap);
}
rowNum++;
}
excelMap.put(key, data);
}
// 关闭文件输入流和工作簿对象
file.close();
workbook.close();
} catch (IOException e) {
e.printStackTrace();
}
return excelMap;
}
/**
* 将List<Map<String, String>>写入Excel文件中
*
* @param data 要写入Excel的数据每个Map代表一行数据Map的key为列名value为单元格数据
* @param excelFilePath Excel文件路径包含文件名和扩展名
* @param sheetName 工作表名称
* @throws IOException 如果写入Excel文件时发生IO异常则抛出该异常
*/
public static void write(List<Map<String, String>> data, String excelFilePath, String sheetName) throws IOException {
// 创建一个新的工作簿对象
Workbook workbook = new XSSFWorkbook();
// 创建一个新的工作表
Sheet sheet = workbook.createSheet(sheetName);
// 行码
int rowNum = 0;
// 写入列头
Row headerRow = sheet.createRow(rowNum++);
int colNum = 0;
for (String key : data.get(0).keySet()) {
Cell cell = headerRow.createCell(colNum++);
cell.setCellValue(key);
}
// 写入数据
for (Map<String, String> rowMap : data) {
Row row = sheet.createRow(rowNum++);
colNum = 0;
for (String key : rowMap.keySet()) {
Cell cell = row.createCell(colNum++);
try {
String s = rowMap.get(key);
if (s.length() > 30000) {
cell.setCellValue(s.substring(0, 25000));
} else {
cell.setCellValue(s);
}
} catch (Exception e) {
System.out.println(key);
e.printStackTrace();
}
}
}
// 将数据写入文件
FileOutputStream outputStream = new FileOutputStream(excelFilePath);
workbook.write(outputStream);
workbook.close();
outputStream.close();
}
// public static void copyFile(String sourceFloder, String targetFileName) {
// File sourceFile = new File(sourceFloder);
// byte[] buffer = new byte[(int) sourceFile.length()];
// try (InputStream inputStream = new FileInputStream(sourceFile)) {
// inputStream.read(buffer);
// } catch (IOException e) {
// e.printStackTrace();
// return;
// }
// // 写入目标文件
// File targetFile = new File(targetFileName);
// targetFile.mkdirs();
// try (OutputStream outputStream = new FileOutputStream(targetFile)) {
// outputStream.write(buffer);
// } catch (IOException e) {
// e.printStackTrace();
// return;
// }
// }
public static void copyFile(String sourceFilePath) {
// 源文件和目标文件的路径
String targetDrive = "F:";
try {
// 获取源文件和目标文件的路径信息
Path sourcePath = Paths.get(sourceFilePath);
Path targetPath = Paths.get(targetDrive + sourcePath.toString().substring(2));
// 如果目标文件的父目录不存在则创建该目录
if (!targetPath.getParent().toFile().exists()) {
targetPath.getParent().toFile().mkdirs();
}
// 进行文件复制
Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING);
System.out.println("Copied file: " + sourceFilePath + " -> " + targetPath);
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
System.out.println(parse(new File("D:\\fromHanler\\xls\\b51484b213ed8fea61f5b99cbdc1490e.xlsx")));
}
}

117
src/main/java/com/bfd/crawl/ppthandler/util/FileDownloader.java

@ -0,0 +1,117 @@
package com.bfd.crawl.ppthandler.util;
import com.alibaba.fastjson.JSON;
import okhttp3.*;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
/**
* @author:jinming
* @className:FileDownloader
* @version:1.0
* @description:
* @Date:2023/8/1 16:31
*/
public class FileDownloader {
private static OkHttpClient okHttpClient;
private static OkHttpClient getOkHttpClient() {
if (okHttpClient == null) {
okHttpClient = new OkHttpClient();
}
return okHttpClient;
}
public static void downloadFile(String url, File destination) throws IOException {
OkHttpClient client = getOkHttpClient();
Request request = new Request.Builder()
.url(url)
.build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("Failed to download file: " + response);
}
ResponseBody body = response.body();
if (body == null) {
throw new IOException("Response body is null");
}
if (!destination.getParentFile().exists()) {
destination.getParentFile().mkdirs();
}
try (InputStream inputStream = body.byteStream();
FileOutputStream outputStream = new FileOutputStream(destination)) {
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
outputStream.flush();
}
}
}
public static Map<String, String> uploadFile(String url, String filePath) throws Exception {
File file = new File(filePath);
Map returnMap = new HashMap(32);
OkHttpClient client = getOkHttpClient();
// 设置文件上传的媒体类型
MediaType mediaType = MediaType.parse("application/octet-stream");
// 创建请求体将文件添加到请求体中
RequestBody requestBody = RequestBody.create(mediaType, file);
// 创建多部分请求体用于上传文件
MultipartBody multipartBody = new MultipartBody.Builder()
.setType(MultipartBody.FORM)
.addFormDataPart("file", file.getName(), requestBody)
.build();
// 创建上传文件的请求
Request request = new Request.Builder()
.url(url)
.post(multipartBody)
.build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("Failed to upload file: " + response);
}
String html = response.body().string();
try {
Map parse = (Map) JSON.parse(html);
Map data = (Map) parse.get("data");
String domain = (String) data.get("domain");
String src = (String) data.get("src");
String fileUrl = domain.concat(src);
returnMap.put("fileUrl", fileUrl);
} catch (Exception e) {
returnMap.put("fileUrl", html);
}
// 处理上传成功的响应
System.out.println("File uploaded successfully!");
}
return returnMap;
}
public static void deleteFile(String url, String md5) throws Exception {
OkHttpClient client = getOkHttpClient();
url = url.concat("delete?md5=").concat(md5);
Request request = new Request.Builder()
.url(url)
.get()
.build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("Failed to upload file: " + response);
}
}
}
}

42
src/main/java/com/bfd/crawl/ppthandler/util/FileUtil.java

@ -0,0 +1,42 @@
package com.bfd.crawl.ppthandler.util;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
/**
* @author:jinming
* @className:FileUtil
* @version:1.0
* @description:
* @Date:2023/8/2 10:57
*/
public class FileUtil {
public static void main(String[] args) {
System.out.println(traverseAndReturnFilePath("D:\\\\ocr\\\\305ce27d7a05770456fdc09d0b3044f7\\\\"));
}
public static List<String> traverseAndReturnFilePath(String folderPath) {
List<String> fileList = new ArrayList<>();
File folder = new File(folderPath);
// 检查文件夹是否存在并且是一个文件夹
if (folder.exists() && folder.isDirectory()) {
// 获取文件夹中的所有文件和子文件夹
File[] files = folder.listFiles();
if (files != null) {
for (File file : files) {
if (file.isFile()) {
// 如果是文件则输出全路径
fileList.add(file.getAbsolutePath());
} else if (file.isDirectory()) {
}
}
}
} else {
System.out.println("指定的路径不是一个文件夹或文件夹不存在。");
}
return fileList;
}
}

59
src/main/java/com/bfd/crawl/ppthandler/util/OcrUtil.java

@ -0,0 +1,59 @@
package com.bfd.crawl.ppthandler.util;
import com.alibaba.fastjson.JSON;
import okhttp3.*;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* @author:jinming
* @className:ocrUtil
* @version:1.0
* @description:
* @Date:2023/8/1 16:38
*/
public class OcrUtil {
private static OkHttpClient okHttpClient;
private static OkHttpClient getOkHttpClient() {
if (okHttpClient == null) {
okHttpClient = new OkHttpClient();
}
return okHttpClient;
}
public static String doOcr(String url,String ocrApi) {
String text = "";
int reTryTimes = 3;
for (int i = 0; i < reTryTimes; i++) {
int okCode = 200;
OkHttpClient client = getOkHttpClient();
OkHttpClient.Builder builder = client.newBuilder().writeTimeout(600, TimeUnit.SECONDS).connectTimeout(600, TimeUnit.SECONDS).readTimeout(600, TimeUnit.SECONDS);
client = builder.build();
MediaType mediaType = MediaType.parse("application/json");
RequestBody body = RequestBody.create(mediaType, "{\"id\":\"\",\"url\":\"" + url + "\"}");
Request request = new Request.Builder()
.url(ocrApi)
.method("POST", body)
.addHeader("Content-Type", "application/json")
.build();
try {
Response response = client.newCall(request).execute();
String html = response.body().string();
Map dataMap = (Map) JSON.parse(html);
int code = (int) dataMap.get("code");
if (code == okCode) {
text = (String) dataMap.get("text");
}
if (StringUtil.hasValue(text)) {
break;
}
} catch (Exception e) {
e.printStackTrace();
}
}
return text;
}
}

23
src/main/java/com/bfd/crawl/ppthandler/util/OsUtil.java

@ -0,0 +1,23 @@
package com.bfd.crawl.ppthandler.util;
import lombok.extern.slf4j.Slf4j;
import java.util.Locale;
/**
* @author:jinming
* @className:OsUtil
* @version:1.0
* @description:
* @Date:2023/4/23 9:40
*/
@Slf4j
public class OsUtil {
public static Boolean isWindows() {
String windows = "windows";
String osName = System.getProperty("os.name").toLowerCase(Locale.ROOT);
log.info("osName = " + osName);
return osName.contains(windows);
}
}

124
src/main/java/com/bfd/crawl/ppthandler/util/PptUtil.java

@ -0,0 +1,124 @@
package com.bfd.crawl.ppthandler.util;
import org.apache.poi.hslf.usermodel.HSLFShape;
import org.apache.poi.hslf.usermodel.HSLFSlide;
import org.apache.poi.hslf.usermodel.HSLFSlideShow;
import org.apache.poi.hslf.usermodel.HSLFTextShape;
import org.apache.poi.xslf.usermodel.*;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* @author:jinming
* @className:PptUtil
* @version:1.0
* @description:
* @Date:2024/3/25 16:14
*/
public class PptUtil {
public static String parsepptx(String filePath) {
StringBuilder dataStringsb = new StringBuilder();
XMLSlideShow ppt = null;
try {
// PPT类
ppt = new XMLSlideShow(new FileInputStream(filePath));
// 获取PPT中的所有幻灯片
// List<XSLFSlide> slides = ppt.getSlides();
// 遍历幻灯片
for (int i = 0; i < ppt.getSlides().size(); i++) {
//拿到第i页的PPT
XSLFSlide slides = ppt.getSlides().get(i);
System.out.println("第" + (i + 1) + "页");
//注释的for循环是获取所以PPT的内容
// for (XSLFSlide slide : slides) {
// 获取幻灯片中的所有图形
List<XSLFShape> shapes = slides.getShapes();
// 遍历PPT的图形
for (XSLFShape shape : shapes) {
// 判断该图形类是否是文本框类
if (shape instanceof XSLFTextShape) {
// 将图像类强制装换成文本框类
XSLFTextShape ts = (XSLFTextShape) shape;
// 获取文本框内的文字
String str = ts.getText();
dataStringsb.append(str);
}
// 判断该图形类是否是表格类
if (shape instanceof XSLFTable) {
// 将图像类强制装换成表格类
XSLFTable table = (XSLFTable) shape;
// 获取表格中的所有行
List<XSLFTableRow> rows = table.getRows();
for (XSLFTableRow tr : rows) {
// 获取行中的所有单元格
List<XSLFTableCell> cells = tr.getCells();
for (XSLFTableCell tc : cells) {
// 获取单元格内的文字
String str = tc.getText();
dataStringsb.append(str);
}
}
}
// 判断该图形类是否是图片框类chr
if (shape instanceof XSLFPictureShape) {
// 将图像类强制装换成图片框类
XSLFPictureShape ps = (XSLFPictureShape) shape;
// 获取图片的字节码数据可以利用输出流将该图片保存到硬盘里
byte[] pictureData = ps.getPictureData().getData();
// System.out.println("图片信息:" + pictureData);
}
}
}
// }
} catch (Exception e) {
e.printStackTrace();
} finally {
if (ppt != null) {
try {
// 保存完之后要对PPT进行关闭操作
ppt.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return dataStringsb.toString();
}
public static String parsePpt(String filePath) {
StringBuilder dataStringsb = new StringBuilder();
HSLFSlideShow ppt = null;
try {
ppt = new HSLFSlideShow(new FileInputStream(filePath));
for (HSLFSlide slide : ppt.getSlides()) {
for (HSLFShape shape : slide.getShapes()) {
if (shape instanceof HSLFTextShape) {
HSLFTextShape textShape = (HSLFTextShape) shape;
dataStringsb.append(textShape.getText()).append("\n");
}
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (ppt != null) {
try {
ppt.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return dataStringsb.toString();
}
}

19
src/main/java/com/bfd/crawl/ppthandler/util/QueueUtil.java

@ -0,0 +1,19 @@
package com.bfd.crawl.ppthandler.util;
import java.util.concurrent.LinkedBlockingDeque;
/**
* @author:jinming
* @className:QueueUtil
* @version:1.0
* @description:
* @Date:2023/7/13 15:00
*/
public class QueueUtil {
public static LinkedBlockingDeque<String> taskQueue = new LinkedBlockingDeque<String>();
public static LinkedBlockingDeque<String> sendQueue = new LinkedBlockingDeque<String>();
}

94
src/main/java/com/bfd/crawl/ppthandler/util/StringUtil.java

@ -0,0 +1,94 @@
package com.bfd.crawl.ppthandler.util;
import lombok.extern.slf4j.Slf4j;
import java.security.MessageDigest;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author jinming
* @version 1.0
* @className StringUtile
* @Date 2022/1/21 11:46
*/
@Slf4j
public class StringUtil {
public static boolean hasValue(String str) {
return str != null && !"".equals(str.trim());
}
public static String getRegexGroup(String regex, String str, int id) {
String resultStr = "";
if (hasValue(str)) {
Pattern p = Pattern.compile(regex);
Matcher m = p.matcher(str);
if (m.find()) {
resultStr = m.group(id);
}
}
if ("".equals(resultStr)) {
}
return resultStr;
}
public static Set<String> getEmailAddress(String message) {
Set<String> emailList = new HashSet<>();
Pattern pattern = Pattern.compile("\\w+\\.?\\w+\\@\\w+\\.\\w+");
Matcher m = pattern.matcher(message);
while (m.find()) {
emailList.add(m.group(0));
}
return emailList;
}
public static String getMd5(String string) {
try {
MessageDigest md5 = MessageDigest.getInstance("MD5");
byte[] bs = md5.digest(string.getBytes("UTF-8"));
StringBuilder sb = new StringBuilder(40);
for (byte x : bs) {
if ((x & 0xff) >> 4 == 0) {
sb.append("0").append(Integer.toHexString(x & 0xff));
} else {
sb.append(Integer.toHexString(x & 0xff));
}
}
return sb.toString();
} catch (Exception e) {
//LOG.error("获取md5异常", e);
return "nceaform" + System.currentTimeMillis();
}
}
public static String removeAllHtmlTags(String str) {
return hasValue(str) ? str.replaceAll("<[^<>]+?>", "") : "";
}
public static String getRegexGroup(Pattern regex, String str, int id) {
String resultStr = "";
if (hasValue(str)) {
Matcher m = regex.matcher(str);
if (m.find()) {
resultStr = m.group(id);
}
}
if ("".equals(resultStr)) {
log.error(regex + " parser error!");
}
return resultStr;
}
public static String getStrByPattern(String str, String regex) {
Pattern pattern = Pattern.compile(regex);
Matcher m = pattern.matcher(str);
return m.find() ? m.group(0) : "";
}
}

47
src/main/resources/application.yml

@ -0,0 +1,47 @@
server:
port: 7082
spring:
application:
name: 幻灯片处理
boot:
admin:
client:
health:
timeout: 10s
url: http://172.16.12.55:8001
instance:
service-base-url: http://172.16.12.55:7088
kafka:
bootstrap-servers: 172.26.28.30:9092
producer:
retries: 3
acks: all
batch-size: 4096
buffer-memory: 102476800
key-serializer: org.apache.kafka.common.serialization.StringSerializer
value-serializer: org.apache.kafka.common.serialization.StringSerializer
logging:
file:
path: ./logs
management:
endpoints:
web:
exposure:
include: "*"
endpoint:
health:
show-details: always
send:
topic: analyze0912
file:
path: D:\\fromHanler\\
thread:
handler: 1
send: 1

36
src/main/resources/logback-spring.xml

@ -0,0 +1,36 @@
<configuration>
<!-- 属性文件:在properties文件中找到对应的配置项 -->
<springProperty scope="context" name="logging.file.path" source="logging.file.path"/>
<springProperty scope="context" name="logging.level" source="logging.level"/>
<!-- 默认的控制台日志输出,一般生产环境都是后台启动,这个没太大作用 -->
<appender name="STDOUT"
class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</pattern>
</encoder>
</appender>
<appender name="GLMAPPER-LOGGERONE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<append>true</append>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>${logging.level}</level>
</filter>
<file>
${logging.file.path}/formHandler.log
</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${logging.file.path}/formHandler.log.%d{yyyy-MM-dd}</FileNamePattern>
<MaxHistory>3</MaxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<root level="info">
<appender-ref ref="GLMAPPER-LOGGERONE"/>
<appender-ref ref="STDOUT"/>
</root>
</configuration>
Loading…
Cancel
Save