package com.bfd.task.process; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import com.alibaba.fastjson.JSONObject; import com.bfd.task.cache.ConfigCache; import com.bfd.task.entity.Constants; import com.bfd.task.utils.QueueUtil; import java.time.Duration; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; /** * @author jian.mao * @date 2023年9月21日 * @description */ @Slf4j @Component public class DataConsumptionProcess implements Runnable { @Value("${spring.kafka.bootstrap-servers}") private String bootstrapServers; @Value("${spring.kafka.consumer.group-id}") private String groupId; @Value("${customize-kafka.consumer.topic}") private String topic; @SuppressWarnings("unchecked") @Override public void run() { // 创建 Kafka 消费者配置 Map consumerProps = new HashMap(16); consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); //跟读 consumerProps.put("auto.offset.reset", "latest"); consumerProps.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); consumerProps.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); consumerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); consumerProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); Consumer consumer = new KafkaConsumer<>(consumerProps); try { // 订阅主题 consumer.subscribe(Collections.singletonList(topic)); // 消费消息 while (true) { // 没超时的话正常消费数据 ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); for (ConsumerRecord record : records) { Map resultData = new HashMap(32); try { // 处理消息,这里可以根据需要进行业务处理 Map resultEs = JSONObject.parseObject(record.value()); log.info("Received message: "+ record.value()); if(!resultEs.containsKey(Constants.TASKID)){ log.warn("数据体缺少taskId"); continue; } String taskId = resultEs.get(Constants.TASKID).toString(); if(!ConfigCache.taskCache.containsKey(taskId)){ log.warn("不属于有知任务产出的数据,taskId:{}",taskId); continue; } Map task = (Map) ConfigCache.taskCache.get(taskId); String token = (String) task.get(Constants.BUSINESSKEY); Map input = (Map) task.get(Constants.INPUT); Integer hasVideo = (Integer) input.get(Constants.HASVIDEO); if(resultEs.get(Constants.HASVIDEO).equals(hasVideo)){ Map crawlResults = JSONObject.parseObject(record.value()); //结果加工 例如videopath[]转换成String bulidResult(crawlResults); // 结果集组装 Map result = new HashMap(16); //结果内容 Map data = new HashMap(16); //获取输出字段 Map output = (Map) task.get(Constants.OUTPUT); for (String key: output.keySet()) { if (crawlResults.containsKey(key)){ data.put(key,crawlResults.get(key)); } } result.put(Constants.RESULTS, JSONObject.toJSONString(data)); for (String key : task.keySet()) { resultData.put(key, task.get(key)); } result.put(Constants.STATUS, 1); result.put(Constants.MESSAGE, "成功"); resultData.put(Constants.RESULT, result); QueueUtil.sendQueue.put(JSONObject.toJSONString(resultData)); //taskId赋值 if(resultEs.containsKey(Constants.TASKID)){ if(taskId == null){ taskId = resultEs.get(Constants.TASKID).toString(); } } }else{ log.info("不符合需求数据----"); } } catch (Exception e) { // TODO: handle exception log.error("数据格式异常:{}",record.value()); //结果集 Map result = new HashMap(16); //遍历入库返回结果,拼接响应内容 result.put(Constants.RESULTS, e.getMessage()); result.put(Constants.MESSAGE, "异常"); result.put(Constants.STATUS, 2); resultData.put(Constants.RESULT, result); //发送kafka QueueUtil.sendQueue.put(JSONObject.toJSONString(resultData)); } } } } catch (Exception e) { log.error("kafka消费异常\n", e); consumer.close(); } } /** * 结果加工 * @param result */ private void bulidResult(Map result){ //视频gofast地址加工 List videoPath = (List) result.get(Constants.VIDEOPATH); if(videoPath != null && videoPath.size() > 0){ String videoUrl = videoPath.get(0); result.put(Constants.VIDEOPATH, videoUrl); } } }