commit 6d632b6f4717d83c964c7e15cff2cdad7dcefbb0 Author: maojian <550076202@qq.com> Date: Fri Dec 12 18:22:51 2025 +0800 nvps rag智能检索 diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..20b2042 --- /dev/null +++ b/.classpath @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..c102fec --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + RagEngine + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..839d647 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..71df522 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,9 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.methodParameters=generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/logs/ragEngineInfo.log b/logs/ragEngineInfo.log new file mode 100644 index 0000000..cd9977d --- /dev/null +++ b/logs/ragEngineInfo.log @@ -0,0 +1,242 @@ +2025-11-07 18:21:17.669 [main] 55 INFO com.bw.search.Application - Starting Application on maojian with PID 72232 (D:\eclipseWork\RagEngine\target\classes started by 55007 in D:\eclipseWork\RagEngine) +2025-11-07 18:21:17.676 [main] 651 INFO com.bw.search.Application - No active profile set, falling back to default profiles: default +2025-11-07 18:21:17.719 [background-preinit] 127 WARN o.s.h.converter.json.Jackson2ObjectMapperBuilder - For Jackson Kotlin classes support please add "com.fasterxml.jackson.module:jackson-module-kotlin" to the classpath +2025-11-07 18:21:18.738 [main] 92 INFO o.s.boot.web.embedded.tomcat.TomcatWebServer - Tomcat initialized with port(s): 8001 (http) +2025-11-07 18:21:18.744 [main] 173 INFO org.apache.coyote.http11.Http11NioProtocol - Initializing ProtocolHandler ["http-nio-8001"] +2025-11-07 18:21:18.744 [main] 173 INFO org.apache.catalina.core.StandardService - Starting service [Tomcat] +2025-11-07 18:21:18.744 [main] 173 INFO org.apache.catalina.core.StandardEngine - Starting Servlet engine: [Apache Tomcat/9.0.30] +2025-11-07 18:21:18.816 [main] 173 INFO o.a.c.c.ContainerBase.[Tomcat].[localhost].[/nvps] - Initializing Spring embedded WebApplicationContext +2025-11-07 18:21:18.816 [main] 284 INFO org.springframework.web.context.ContextLoader - Root WebApplicationContext: initialization completed in 1094 ms +2025-11-07 18:21:19.120 [main] 171 INFO o.s.scheduling.concurrent.ThreadPoolTaskExecutor - Initializing ExecutorService 'applicationTaskExecutor' +2025-11-07 18:21:19.302 [main] 171 INFO o.s.scheduling.concurrent.ThreadPoolTaskScheduler - Initializing ExecutorService 'taskScheduler' +2025-11-07 18:21:19.307 [main] 58 INFO o.s.b.actuate.endpoint.web.EndpointLinksResolver - Exposing 2 endpoint(s) beneath base path '/actuator' +2025-11-07 18:21:19.363 [main] 347 INFO org.apache.kafka.clients.admin.AdminClientConfig - AdminClientConfig values: + bootstrap.servers = [node-01:19092, node-02:19092, node-03:19092] + client.dns.lookup = default + client.id = + connections.max.idle.ms = 300000 + metadata.max.age.ms = 300000 + metric.reporters = [] + metrics.num.samples = 2 + metrics.recording.level = INFO + metrics.sample.window.ms = 30000 + receive.buffer.bytes = 65536 + reconnect.backoff.max.ms = 1000 + reconnect.backoff.ms = 50 + request.timeout.ms = 120000 + retries = 5 + retry.backoff.ms = 100 + sasl.client.callback.handler.class = null + sasl.jaas.config = null + sasl.kerberos.kinit.cmd = /usr/bin/kinit + sasl.kerberos.min.time.before.relogin = 60000 + sasl.kerberos.service.name = null + sasl.kerberos.ticket.renew.jitter = 0.05 + sasl.kerberos.ticket.renew.window.factor = 0.8 + sasl.login.callback.handler.class = null + sasl.login.class = null + sasl.login.refresh.buffer.seconds = 300 + sasl.login.refresh.min.period.seconds = 60 + sasl.login.refresh.window.factor = 0.8 + sasl.login.refresh.window.jitter = 0.05 + sasl.mechanism = GSSAPI + security.protocol = PLAINTEXT + send.buffer.bytes = 131072 + ssl.cipher.suites = null + ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1] + ssl.endpoint.identification.algorithm = https + ssl.key.password = null + ssl.keymanager.algorithm = SunX509 + ssl.keystore.location = null + ssl.keystore.password = null + ssl.keystore.type = JKS + ssl.protocol = TLS + ssl.provider = null + ssl.secure.random.implementation = null + ssl.trustmanager.algorithm = PKIX + ssl.truststore.location = null + ssl.truststore.password = null + ssl.truststore.type = JKS + +2025-11-07 18:21:19.416 [main] 117 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka version: 2.3.1 +2025-11-07 18:21:19.417 [main] 118 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka commitId: 18a913733fb71c01 +2025-11-07 18:21:19.417 [main] 119 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka startTimeMs: 1762510879415 +2025-11-07 18:21:19.705 [main] 347 INFO org.apache.kafka.clients.consumer.ConsumerConfig - ConsumerConfig values: + allow.auto.create.topics = true + auto.commit.interval.ms = 1000 + auto.offset.reset = earliest + bootstrap.servers = [node-01:19092, node-02:19092, node-03:19092] + check.crcs = true + client.dns.lookup = default + client.id = + client.rack = + connections.max.idle.ms = 540000 + default.api.timeout.ms = 60000 + enable.auto.commit = true + exclude.internal.topics = true + fetch.max.bytes = 52428800 + fetch.max.wait.ms = 500 + fetch.min.bytes = 1 + group.id = test4 + group.instance.id = null + heartbeat.interval.ms = 3000 + interceptor.classes = [] + internal.leave.group.on.close = true + isolation.level = read_uncommitted + key.deserializer = class org.apache.kafka.common.serialization.StringDeserializer + max.partition.fetch.bytes = 1048576 + max.poll.interval.ms = 300000 + max.poll.records = 500 + metadata.max.age.ms = 300000 + metric.reporters = [] + metrics.num.samples = 2 + metrics.recording.level = INFO + metrics.sample.window.ms = 30000 + partition.assignment.strategy = [class org.apache.kafka.clients.consumer.RangeAssignor] + receive.buffer.bytes = 65536 + reconnect.backoff.max.ms = 1000 + reconnect.backoff.ms = 50 + request.timeout.ms = 30000 + retry.backoff.ms = 100 + sasl.client.callback.handler.class = null + sasl.jaas.config = null + sasl.kerberos.kinit.cmd = /usr/bin/kinit + sasl.kerberos.min.time.before.relogin = 60000 + sasl.kerberos.service.name = null + sasl.kerberos.ticket.renew.jitter = 0.05 + sasl.kerberos.ticket.renew.window.factor = 0.8 + sasl.login.callback.handler.class = null + sasl.login.class = null + sasl.login.refresh.buffer.seconds = 300 + sasl.login.refresh.min.period.seconds = 60 + sasl.login.refresh.window.factor = 0.8 + sasl.login.refresh.window.jitter = 0.05 + sasl.mechanism = GSSAPI + security.protocol = PLAINTEXT + send.buffer.bytes = 131072 + session.timeout.ms = 10000 + ssl.cipher.suites = null + ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1] + ssl.endpoint.identification.algorithm = https + ssl.key.password = null + ssl.keymanager.algorithm = SunX509 + ssl.keystore.location = null + ssl.keystore.password = null + ssl.keystore.type = JKS + ssl.protocol = TLS + ssl.provider = null + ssl.secure.random.implementation = null + ssl.trustmanager.algorithm = PKIX + ssl.truststore.location = null + ssl.truststore.password = null + ssl.truststore.type = JKS + value.deserializer = class org.apache.kafka.common.serialization.StringDeserializer + +2025-11-07 18:21:19.724 [main] 117 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka version: 2.3.1 +2025-11-07 18:21:19.724 [main] 118 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka commitId: 18a913733fb71c01 +2025-11-07 18:21:19.724 [main] 119 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka startTimeMs: 1762510879724 +2025-11-07 18:21:19.724 [main] 964 INFO org.apache.kafka.clients.consumer.KafkaConsumer - [Consumer clientId=consumer-1, groupId=test4] Subscribed to topic(s): nvps_search_data +2025-11-07 18:21:19.726 [main] 171 INFO o.s.scheduling.concurrent.ThreadPoolTaskScheduler - Initializing ExecutorService +2025-11-07 18:21:19.737 [main] 173 INFO org.apache.coyote.http11.Http11NioProtocol - Starting ProtocolHandler ["http-nio-8001"] +2025-11-07 18:21:19.745 [main] 204 INFO o.s.boot.web.embedded.tomcat.TomcatWebServer - Tomcat started on port(s): 8001 (http) with context path '/nvps' +2025-11-07 18:21:19.747 [main] 61 INFO com.bw.search.Application - Started Application in 2.35 seconds (JVM running for 2.636) +2025-11-07 18:21:20.058 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 261 INFO org.apache.kafka.clients.Metadata - [Consumer clientId=consumer-1, groupId=test4] Cluster ID: NyNtqsIDRIae-HJZGe2pww +2025-11-07 18:21:20.087 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 728 INFO o.a.k.c.consumer.internals.AbstractCoordinator - [Consumer clientId=consumer-1, groupId=test4] Discovered group coordinator node-03:19092 (id: 2147483644 rack: null) +2025-11-07 18:21:20.089 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 476 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Revoking previously assigned partitions [] +2025-11-07 18:21:20.089 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 279 INFO o.s.kafka.listener.KafkaMessageListenerContainer - test4: partitions revoked: [] +2025-11-07 18:21:20.089 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 505 INFO o.a.k.c.consumer.internals.AbstractCoordinator - [Consumer clientId=consumer-1, groupId=test4] (Re-)joining group +2025-11-07 18:21:20.207 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 505 INFO o.a.k.c.consumer.internals.AbstractCoordinator - [Consumer clientId=consumer-1, groupId=test4] (Re-)joining group +2025-11-07 18:21:20.748 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 469 INFO o.a.k.c.consumer.internals.AbstractCoordinator - [Consumer clientId=consumer-1, groupId=test4] Successfully joined group with generation 23 +2025-11-07 18:21:20.750 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 283 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Setting newly assigned partitions: nvps_search_data-7, nvps_search_data-8, nvps_search_data-1, nvps_search_data-2, nvps_search_data-0, nvps_search_data-5, nvps_search_data-6, nvps_search_data-3, nvps_search_data-4 +2025-11-07 18:21:20.789 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 525 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Setting offset for partition nvps_search_data-7 to the committed offset FetchPosition{offset=0, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=node-03:19092 (id: 3 rack: null), epoch=0}} +2025-11-07 18:21:20.789 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 525 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Setting offset for partition nvps_search_data-8 to the committed offset FetchPosition{offset=1, offsetEpoch=Optional[0], currentLeader=LeaderAndEpoch{leader=node-01:19092 (id: 1 rack: null), epoch=0}} +2025-11-07 18:21:20.790 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 525 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Setting offset for partition nvps_search_data-1 to the committed offset FetchPosition{offset=0, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=node-03:19092 (id: 3 rack: null), epoch=0}} +2025-11-07 18:21:20.790 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 525 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Setting offset for partition nvps_search_data-2 to the committed offset FetchPosition{offset=3, offsetEpoch=Optional[0], currentLeader=LeaderAndEpoch{leader=node-01:19092 (id: 1 rack: null), epoch=0}} +2025-11-07 18:21:20.790 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 525 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Setting offset for partition nvps_search_data-0 to the committed offset FetchPosition{offset=1, offsetEpoch=Optional[0], currentLeader=LeaderAndEpoch{leader=node-02:19092 (id: 2 rack: null), epoch=0}} +2025-11-07 18:21:20.790 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 525 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Setting offset for partition nvps_search_data-5 to the committed offset FetchPosition{offset=0, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=node-01:19092 (id: 1 rack: null), epoch=0}} +2025-11-07 18:21:20.790 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 525 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Setting offset for partition nvps_search_data-6 to the committed offset FetchPosition{offset=0, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=node-02:19092 (id: 2 rack: null), epoch=0}} +2025-11-07 18:21:20.790 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 525 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Setting offset for partition nvps_search_data-3 to the committed offset FetchPosition{offset=1, offsetEpoch=Optional[0], currentLeader=LeaderAndEpoch{leader=node-02:19092 (id: 2 rack: null), epoch=0}} +2025-11-07 18:21:20.790 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 525 INFO o.a.k.c.consumer.internals.ConsumerCoordinator - [Consumer clientId=consumer-1, groupId=test4] Setting offset for partition nvps_search_data-4 to the committed offset FetchPosition{offset=0, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=node-03:19092 (id: 3 rack: null), epoch=0}} +2025-11-07 18:21:20.947 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 279 INFO o.s.kafka.listener.KafkaMessageListenerContainer - test4: partitions assigned: [nvps_search_data-7, nvps_search_data-8, nvps_search_data-1, nvps_search_data-2, nvps_search_data-0, nvps_search_data-5, nvps_search_data-6, nvps_search_data-3, nvps_search_data-4] +2025-11-07 18:21:41.726 [http-nio-8001-exec-2] 173 INFO o.a.c.c.ContainerBase.[Tomcat].[localhost].[/nvps] - Initializing Spring DispatcherServlet 'dispatcherServlet' +2025-11-07 18:21:41.727 [http-nio-8001-exec-2] 525 INFO org.springframework.web.servlet.DispatcherServlet - Initializing Servlet 'dispatcherServlet' +2025-11-07 18:21:41.733 [http-nio-8001-exec-2] 547 INFO org.springframework.web.servlet.DispatcherServlet - Completed initialization in 6 ms +2025-11-07 18:21:41.881 [http-nio-8001-exec-2] 48 INFO com.bw.search.service.impl.RagSearchServiceImpl - 知识库检索开始----- +2025-11-07 18:21:41.895 [http-nio-8001-exec-2] 28 INFO com.bw.search.utils.SpringBootKafka - 准备发送消息为："{\"id\":\"id-1\",\"keyword\":\"Michael J. Aziz\"}" +2025-11-07 18:21:41.902 [http-nio-8001-exec-2] 347 INFO org.apache.kafka.clients.producer.ProducerConfig - ProducerConfig values: + acks = 1 + batch.size = 16384 + bootstrap.servers = [node-01:19092, node-02:19092, node-03:19092] + buffer.memory = 33554432 + client.dns.lookup = default + client.id = + compression.type = none + connections.max.idle.ms = 540000 + delivery.timeout.ms = 120000 + enable.idempotence = false + interceptor.classes = [] + key.serializer = class org.apache.kafka.common.serialization.StringSerializer + linger.ms = 0 + max.block.ms = 60000 + max.in.flight.requests.per.connection = 5 + max.request.size = 1048576 + metadata.max.age.ms = 300000 + metric.reporters = [] + metrics.num.samples = 2 + metrics.recording.level = INFO + metrics.sample.window.ms = 30000 + partitioner.class = class org.apache.kafka.clients.producer.internals.DefaultPartitioner + receive.buffer.bytes = 32768 + reconnect.backoff.max.ms = 1000 + reconnect.backoff.ms = 50 + request.timeout.ms = 30000 + retries = 0 + retry.backoff.ms = 100 + sasl.client.callback.handler.class = null + sasl.jaas.config = null + sasl.kerberos.kinit.cmd = /usr/bin/kinit + sasl.kerberos.min.time.before.relogin = 60000 + sasl.kerberos.service.name = null + sasl.kerberos.ticket.renew.jitter = 0.05 + sasl.kerberos.ticket.renew.window.factor = 0.8 + sasl.login.callback.handler.class = null + sasl.login.class = null + sasl.login.refresh.buffer.seconds = 300 + sasl.login.refresh.min.period.seconds = 60 + sasl.login.refresh.window.factor = 0.8 + sasl.login.refresh.window.jitter = 0.05 + sasl.mechanism = GSSAPI + security.protocol = PLAINTEXT + send.buffer.bytes = 131072 + ssl.cipher.suites = null + ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1] + ssl.endpoint.identification.algorithm = https + ssl.key.password = null + ssl.keymanager.algorithm = SunX509 + ssl.keystore.location = null + ssl.keystore.password = null + ssl.keystore.type = JKS + ssl.protocol = TLS + ssl.provider = null + ssl.secure.random.implementation = null + ssl.trustmanager.algorithm = PKIX + ssl.truststore.location = null + ssl.truststore.password = null + ssl.truststore.type = JKS + transaction.timeout.ms = 60000 + transactional.id = null + value.serializer = class org.apache.kafka.common.serialization.StringSerializer + +2025-11-07 18:21:41.918 [http-nio-8001-exec-2] 117 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka version: 2.3.1 +2025-11-07 18:21:41.919 [http-nio-8001-exec-2] 118 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka commitId: 18a913733fb71c01 +2025-11-07 18:21:41.919 [http-nio-8001-exec-2] 119 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka startTimeMs: 1762510901918 +2025-11-07 18:21:42.058 [kafka-producer-network-thread | producer-1] 261 INFO org.apache.kafka.clients.Metadata - [Producer clientId=producer-1] Cluster ID: NyNtqsIDRIae-HJZGe2pww +2025-11-07 18:21:42.068 [http-nio-8001-exec-2] 116 INFO com.bw.search.service.impl.RagSearchServiceImpl - 知识获取中请稍后... +2025-11-07 18:21:42.175 [kafka-producer-network-thread | producer-1] 41 INFO com.bw.search.utils.SpringBootKafka - nvps_know_base - 生产者 发送消息成功：SendResult [producerRecord=ProducerRecord(topic=nvps_know_base, partition=null, headers=RecordHeaders(headers = [], isReadOnly = true), key=null, value={"id":"id-1","keyword":"Michael J. Aziz"}, timestamp=null), recordMetadata=nvps_know_base-7@1] +2025-11-07 18:21:43.085 [http-nio-8001-exec-2] 116 INFO com.bw.search.service.impl.RagSearchServiceImpl - 知识获取中请稍后... +2025-11-07 18:21:44.092 [http-nio-8001-exec-2] 116 INFO com.bw.search.service.impl.RagSearchServiceImpl - 知识获取中请稍后... +2025-11-07 18:21:45.106 [http-nio-8001-exec-2] 116 INFO com.bw.search.service.impl.RagSearchServiceImpl - 知识获取中请稍后... +2025-11-07 18:21:45.770 [org.springframework.kafka.KafkaListenerEndpointContainer#0-0-C-1] 35 INFO com.bw.search.process.ResultSendQueue - 消费知识：id-1 +2025-11-07 18:21:46.128 [http-nio-8001-exec-2] 51 INFO com.bw.search.service.impl.RagSearchServiceImpl - 知识库结果已获取：{"isLast":1,"promptRes":"已知信息：\n{\"college\":\"Bertoldi Group: Solid Mechanics\",\"phone\":\"(617) 496-3084\",\"researchFocus\":[\"Modeling Physical/Biological Phenomena and Systems\",\"Materials\",\"Materials\",\"Solid Mechanics\"],\"name\":\"Katia Bertoldi\",\"position\":\"William and Ami Kuan Danoff Professor of Applied Mechanics\",\"email\":\"bertoldi@seas.harvard.edu\"}\n{\"college\":\"\",\"phone\":\"(617) 495-6012\",\"researchFocus\":[\"Materials\",\"Robotics and Control\",\"Robotics and Control\",\"Materials\",\"Robotics and Control\",\"Design and Innovation\"],\"name\":\"Martin Bechthold\",\"position\":\"Kumagai Professor of Architectural Technology; Affiliate in Materials Science & Mechanical Engineering\",\"email\":\"mbechthold@gsd.harvard.edu\"}\n{\"college\":\"Anderson Group\",\"phone\":\"(617) 998-5550\",\"researchFocus\":[\"Atmospheric Chemistry\",\"Climate Change\",\"Solar Geoengineering\"],\"name\":\"James G. Anderson\",\"position\":\"Philip S. Weld Professor of Atmospheric Chemistry\",\"email\":\"anderson@huarp.harvard.edu\"}\n{\"college\":\"Markus Basan Lab\",\"phone\":\"\",\"researchFocus\":[\"Modeling Physical/Biological Phenomena and Systems\",\"Soft Matter\",\"Soft Matter\"],\"name\":\"Markus Thomas Basan\",\"position\":\"Assistant Professor of Systems Biology; Affiliate in Applied Physics\",\"email\":\"markus@hms.harvard.edu\"}\n{\"college\":\"\",\"phone\":\"\",\"researchFocus\":[\"Modeling Physical/Biological Phenomena and Systems\"],\"name\":\"William H. Bossert\",\"position\":\"David B. Arnold, Jr. Professor of Science, Emeritus\",\"email\":\"bossert@seas.harvard.edu\"}\n\n参考上述已知信息回答问题：\nMichael J. Aziz","ids":"[\"7dc72740-bbc3-11f0-9abc-00163e03f4d5\", \"7a5654fa-bbc3-11f0-9abc-00163e03f4d5\", \"783b6c46-bbc3-11f0-9abc-00163e03f4d5\", \"78ac3a02-bbc3-11f0-9abc-00163e03f4d5\", \"7d5e213c-bbc3-11f0-9abc-00163e03f4d5\"]","id":"id-1","keyword":"Michael J. Aziz"} +2025-11-07 18:21:46.141 [http-nio-8001-exec-2] 98 INFO com.bw.search.service.impl.RagSearchServiceImpl - 模型请求信息：url:https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions,header:{"authorization":"Bearer sk-c5f56c1c8a6447b3a6c646a3f14085c4","Content-Type":"application/json"},params：{"top_p":0.95,"stream":false,"max_tokens":8192,"temperature":0.7,"messages":[{"role":"user","content":"You are a professional academic profile writer.Task:You will receive multiple lines of text. Each line contains either:- A plain JSON object with researcher information, OR - A wrapper object that includes a \"value\" field containing the actual researcher JSON string.For each line, extract the researcher information and generate a concise academic biography in English (3–5 sentences). Avoid listing items mechanically — make it sound natural and professional. If some fields (like phone or email) are missing, just skip them. Output one biography paragraph per person, separated by blank lines.Input example:{\"college\":\"Aizenberg Biomineralization and Biomimetics Lab\",\"phone\":\"(617) 495-3558\",\"researchFocus\":[\"Biomaterials\",\"Soft Matter\",\"Surface and Interface Science\"],\"name\":\"Joanna Aizenberg\",\"position\":\"Amy Smith Berylson Professor of Materials Science and Professor of Chemistry & Chemical Biology\",\"email\":\"jaiz@seas.harvard.edu\"}{\"college\":\"Bertoldi Group: Solid Mechanics\",\"phone\":\"(617) 496-3084\",\"researchFocus\":[\"Modeling Physical/Biological Phenomena and Systems\",\"Materials\",\"Solid Mechanics\"],\"name\":\"Katia Bertoldi\",\"position\":\"William and Ami Kuan Danoff Professor of Applied Mechanics\",\"email\":\"bertoldi@seas.harvard.edu\"}Output format:Write one paragraph per person, separated by a blank line. Each paragraph should summarize their title, affiliation, and main research interests.Now generate the biographies for the following input:已知信息：\n{\"college\":\"Bertoldi Group: Solid Mechanics\",\"phone\":\"(617) 496-3084\",\"researchFocus\":[\"Modeling Physical/Biological Phenomena and Systems\",\"Materials\",\"Materials\",\"Solid Mechanics\"],\"name\":\"Katia Bertoldi\",\"position\":\"William and Ami Kuan Danoff Professor of Applied Mechanics\",\"email\":\"bertoldi@seas.harvard.edu\"}\n{\"college\":\"\",\"phone\":\"(617) 495-6012\",\"researchFocus\":[\"Materials\",\"Robotics and Control\",\"Robotics and Control\",\"Materials\",\"Robotics and Control\",\"Design and Innovation\"],\"name\":\"Martin Bechthold\",\"position\":\"Kumagai Professor of Architectural Technology; Affiliate in Materials Science & Mechanical Engineering\",\"email\":\"mbechthold@gsd.harvard.edu\"}\n{\"college\":\"Anderson Group\",\"phone\":\"(617) 998-5550\",\"researchFocus\":[\"Atmospheric Chemistry\",\"Climate Change\",\"Solar Geoengineering\"],\"name\":\"James G. Anderson\",\"position\":\"Philip S. Weld Professor of Atmospheric Chemistry\",\"email\":\"anderson@huarp.harvard.edu\"}\n{\"college\":\"Markus Basan Lab\",\"phone\":\"\",\"researchFocus\":[\"Modeling Physical/Biological Phenomena and Systems\",\"Soft Matter\",\"Soft Matter\"],\"name\":\"Markus Thomas Basan\",\"position\":\"Assistant Professor of Systems Biology; Affiliate in Applied Physics\",\"email\":\"markus@hms.harvard.edu\"}\n{\"college\":\"\",\"phone\":\"\",\"researchFocus\":[\"Modeling Physical/Biological Phenomena and Systems\"],\"name\":\"William H. Bossert\",\"position\":\"David B. Arnold, Jr. Professor of Science, Emeritus\",\"email\":\"bossert@seas.harvard.edu\"}\n\n参考上述已知信息回答问题：\nMichael J. Aziz"}],"model":"qwen-max-latest"} +2025-11-07 18:22:00.732 [http-nio-8001-exec-2] 61 INFO com.bw.search.service.impl.RagSearchServiceImpl - 问答结果：{"choices":[{"message":{"role":"assistant","content":"Katia Bertoldi is the William and Ami Kuan Danoff Professor of Applied Mechanics at Harvard University, affiliated with the Bertoldi Group in Solid Mechanics. Her research focuses on modeling physical and biological phenomena, materials science, and solid mechanics, contributing significantly to the understanding of complex systems and their behaviors.\n\nMartin Bechthold holds the position of Kumagai Professor of Architectural Technology and is also affiliated with Materials Science and Mechanical Engineering at Harvard. His work centers on advancing materials innovation, robotics and control systems, and design principles, bridging architecture and engineering through cutting-edge technology.\n\nJames G. Anderson serves as the Philip S. Weld Professor of Atmospheric Chemistry in the Anderson Group at Harvard. His pioneering research explores atmospheric chemistry, climate change, and solar geoengineering, addressing critical environmental challenges with far-reaching implications for global sustainability.\n\nMarkus Thomas Basan is an Assistant Professor of Systems Biology with an affiliation in Applied Physics at Harvard Medical School. Leading the Markus Basan Lab, his research investigates the modeling of physical and biological systems, with a particular emphasis on soft matter, contributing to interdisciplinary approaches in life sciences.\n\nWilliam H. Bossert, the David B. Arnold, Jr. Professor of Science, Emeritus, has made significant contributions to the field of modeling physical and biological phenomena. His academic legacy continues to inspire advancements in scientific understanding, even as he steps back from active research.\n\nNo information is available in the provided data about Michael J. Aziz. Additional details would be needed to construct his academic biography."},"finish_reason":"stop","index":0,"logprobs":null}],"object":"chat.completion","usage":{"prompt_tokens":708,"completion_tokens":307,"total_tokens":1015},"created":1762510923,"system_fingerprint":null,"model":"qwen-max-latest","id":"chatcmpl-5751ed60-460e-4e8a-bb07-90cb997e9568"} diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..7ffe66c --- /dev/null +++ b/pom.xml @@ -0,0 +1,237 @@ + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.2.4.RELEASE + + com.bw + RagEngine + 0.0.1-SNAPSHOT + jar + + RagEngine + http://maven.apache.org + + + UTF-8 + 1.8 + 1.8 + + + + + junit + junit + 4.11 + test + + + org.springframework.boot + spring-boot-starter-web + + + + de.codecentric + spring-boot-admin-starter-client + 2.2.4 + + + com.google.code.gson + gson + 2.8.8 + + + org.springframework.boot + spring-boot-test + + + + org.springframework + spring-test + 5.0.10.RELEASE + test + + + org.springframework.kafka + spring-kafka + + + commons-io + commons-io + 2.11.0 + + + + com.alibaba + fastjson + 2.0.17 + + + + com.mchange + c3p0 + 0.9.5.5 + + + + com.squareup.okhttp3 + okhttp + 4.9.3 + + + org.apache.httpcomponents + httpclient + 4.5.3 + + + commons-lang + commons-lang + 2.6 + + + + org.jetbrains.kotlin + kotlin-reflect + 1.6.21 + runtime + + + + + org.apache.logging.log4j + log4j-api + 2.14.1 + + + org.apache.logging.log4j + log4j-core + 2.14.1 + + + + org.projectlombok + lombok + + + cn.hutool + hutool-all + 5.8.5 + + + junit + junit + + + + p6spy + p6spy + 3.9.0 + + + + commons-collections + commons-collections + 3.2.2 + + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + org.springframework.boot + spring-boot-maven-plugin + + com.bw.search.Application + ZIP + + + ${project.groupId} + ${project.artifactId} + + + + + + + repackage + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.1.1 + + + copy + package + + copy-dependencies + + + jar + jar + runtime + ${project.build.directory}/libs + + + + + + + + \ No newline at end of file diff --git a/prompt b/prompt new file mode 100644 index 0000000..45ba0db --- /dev/null +++ b/prompt @@ -0,0 +1,46 @@ +You are a professional academic profile writer. + +### Language Requirement + +Determine the output language dynamically based on the keyword content. +If the keyword is in Arabic, output Arabic. +If the keyword is in Chinese, output Chinese. +If in English, output English. +If ambiguous, default to English. +Use the exact same language for all generated biographies. + +Task + +You will receive multiple lines of text. +Each line contains either: + A plain JSON object with researcher information, OR + A wrapper object that includes a "value" field containing the actual researcher JSON string. + +For each line: + Extract the researcher introduction. + Generate an abstract summary biography (1–3 sentences). + Avoid mechanical listing — write naturally and academically. + If some fields (phone, email, etc.) are missing, simply omit them. + Output one biography paragraph per person. + Insert exactly one blank line between different people’s biographies. + Do not merge multiple people into one paragraph. + +Input Example: + {"college":"Aizenberg Biomineralization and Biomimetics Lab","phone":"(617) 495-3558","researchFocus":["Biomaterials","Soft Matter","Surface and Interface Science"],"name":"Joanna Aizenberg","position":"Amy Smith Berylson Professor of Materials Science and Professor of Chemistry & Chemical Biology","email":"jaiz@seas.harvard.edu"} +{"college":"Bertoldi Group: Solid Mechanics","phone":"(617) 496-3084","researchFocus":["Modeling Physical/Biological Phenomena and Systems","Materials","Solid Mechanics"],"name":"Katia Bertoldi","position":"William and Ami Kuan Danoff Professor of Applied Mechanics","email":"bertoldi@seas.harvard.edu"} + +Output Format: + Name: biography. + + (blank line) + + Name: biography. + +Each biography separated by one empty line. +Written entirely in the language inferred from the keyword. + +keyword（用于自动识别语种）： +${keyword} + +Now generate the biographies for the following input: +${information} \ No newline at end of file diff --git a/prompt-test b/prompt-test new file mode 100644 index 0000000..31ed8b2 --- /dev/null +++ b/prompt-test @@ -0,0 +1,39 @@ +You are a professional academic profile writer. + +Language Requirement + +Determine the output language dynamically based on the keyword content. +If the keyword is in Arabic, output Arabic. +If in English, output English. +If ambiguous, default to English. +Use the exact same language for all generated biographies. + +Task + +You will receive multiple lines of text. +Each line contains either: +A plain JSON object with researcher information, OR +A wrapper object that includes a "value" field containing the actual researcher JSON string. +For each line: +Extract the researcher introduction. +Generate a abstract summary biography (1–3 sentences). +Avoid mechanical listing — write naturally and academically. +If some fields (phone, email, etc.) are missing, simply omit them. +Output one biography paragraph per person, separated by a blank line. + +Input Example +{"college":"Aizenberg Biomineralization and Biomimetics Lab","phone":"(617) 495-3558","researchFocus":["Biomaterials","Soft Matter","Surface and Interface Science"],"name":"Joanna Aizenberg","position":"Amy Smith Berylson Professor of Materials Science and Professor of Chemistry & Chemical Biology","email":"jaiz@seas.harvard.edu +"} +{"college":"Bertoldi Group: Solid Mechanics","phone":"(617) 496-3084","researchFocus":["Modeling Physical/Biological Phenomena and Systems","Materials","Solid Mechanics"],"name":"Katia Bertoldi","position":"William and Ami Kuan Danoff Professor of Applied Mechanics","email":"bertoldi@seas.harvard.edu +"} + +Output Format +Name: biography. +Paragraphs separated by a blank line +Written entirely in the language inferred from keyword + +keyword（用于自动识别语种）： +${keyword} + +Now generate the biographies for the following input: +${information} \ No newline at end of file diff --git a/src/main/java/com/bw/search/Application.java b/src/main/java/com/bw/search/Application.java new file mode 100644 index 0000000..95e0ab7 --- /dev/null +++ b/src/main/java/com/bw/search/Application.java @@ -0,0 +1,25 @@ +package com.bw.search; + + + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * 主入口 + * + * @author jian.mao + * @date 2023年7月4日 + * @description + */ +@SpringBootApplication +@EnableScheduling +@EnableKafka +public class Application { + + public static void main(String[] args) { + SpringApplication.run(Application.class, args); + } +} \ No newline at end of file diff --git a/src/main/java/com/bw/search/cache/ConfigCache.java b/src/main/java/com/bw/search/cache/ConfigCache.java new file mode 100644 index 0000000..3007954 --- /dev/null +++ b/src/main/java/com/bw/search/cache/ConfigCache.java @@ -0,0 +1,22 @@ +package com.bw.search.cache; + +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author jian.mao + * @date 2022年11月11日 + * @description 静态变量类 + */ +@Slf4j +public class ConfigCache { + + /**启动条件**/ + public static boolean isStart = true; + /***知识库信息存储--模型总结使用***/ + public static Map baseResult = new HashMap(16); + /***知识库信息存储--页面查询结果使用***/ + public static Map searchResult = new HashMap(16); +} diff --git a/src/main/java/com/bw/search/common/Res.java b/src/main/java/com/bw/search/common/Res.java new file mode 100644 index 0000000..29460a6 --- /dev/null +++ b/src/main/java/com/bw/search/common/Res.java @@ -0,0 +1,61 @@ +package com.bw.search.common; + +/** + * 通用返回对象 + * + * @author jian.mao + * @date 2025年9月17日 + * @description + * @param + */ + +public class Res { + private int resCode; + private String resMsg; + private T data; + + public Res() { + } + + public Res(int resCode, String resMsg, T data) { + this.resCode = resCode; + this.resMsg = resMsg; + this.data = data; + } + + public static Res ok(T data) { + return new Res(ResponseCode.SUCCESS.code(), ResponseCode.SUCCESS.message(), data); + } + + public static Res fail(String msg) { + return new Res(ResponseCode.FAIL.code(), msg, null); + } + public static Res checkError(T error) { + return new Res(ResponseCode.FAIL.code(), ResponseCode.CHECKERROR.message(), error); + } + + // getter & setter + public int getResCode() { + return resCode; + } + + public void setResCode(int resCode) { + this.resCode = resCode; + } + + public String getResMsg() { + return resMsg; + } + + public void setResMsg(String resMsg) { + this.resMsg = resMsg; + } + + public T getData() { + return data; + } + + public void setData(T data) { + this.data = data; + } +} diff --git a/src/main/java/com/bw/search/common/ResponseCode.java b/src/main/java/com/bw/search/common/ResponseCode.java new file mode 100644 index 0000000..ee4c4c3 --- /dev/null +++ b/src/main/java/com/bw/search/common/ResponseCode.java @@ -0,0 +1,30 @@ +package com.bw.search.common; + +/** + * 状态码管理 + * @author jian.mao + * @date 2025年9月17日 + * @description + */ +public enum ResponseCode { + + SUCCESS(0, "success"), + FAIL(-1, "fail"), + CHECKERROR(400, "paramsError"); + + private final int code; + private final String message; + + ResponseCode(int code, String message) { + this.code = code; + this.message = message; + } + + public int code() { + return code; + } + + public String message() { + return message; + } +} diff --git a/src/main/java/com/bw/search/config/EsConfig.java b/src/main/java/com/bw/search/config/EsConfig.java new file mode 100644 index 0000000..2d6af1f --- /dev/null +++ b/src/main/java/com/bw/search/config/EsConfig.java @@ -0,0 +1,16 @@ +package com.bw.search.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Data +@Component +@ConfigurationProperties(prefix = "es") +public class EsConfig { + + private String host; + private String username; + private String password; + private String index; +} diff --git a/src/main/java/com/bw/search/config/KafkaConfig.java b/src/main/java/com/bw/search/config/KafkaConfig.java new file mode 100644 index 0000000..33e9060 --- /dev/null +++ b/src/main/java/com/bw/search/config/KafkaConfig.java @@ -0,0 +1,20 @@ +package com.bw.search.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +/** + * kafka配置类 + * @author jian.mao + * @date 2023年7月6日 + * @description + */ +@Configuration +public class KafkaConfig { + @Value("${kafka.consumer.topic}") + private String kafkaTopic; + + public String getKafkaTopic() { + return kafkaTopic; + } +} diff --git a/src/main/java/com/bw/search/controller/RagSearchController.java b/src/main/java/com/bw/search/controller/RagSearchController.java new file mode 100644 index 0000000..a343dce --- /dev/null +++ b/src/main/java/com/bw/search/controller/RagSearchController.java @@ -0,0 +1,65 @@ +package com.bw.search.controller; + + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import com.bw.search.common.Res; +import com.bw.search.service.ModelStreamService; +import com.bw.search.service.RagSearchService; + +import lombok.extern.slf4j.Slf4j; + + +@Controller +@CrossOrigin +@RequestMapping("/api") +@Slf4j +public class RagSearchController { + @Autowired + private RagSearchService ragSearchService; + @Autowired + private ModelStreamService modelStreamService; + @PostMapping("/search") + @ResponseBody + public Res search(@RequestBody String dataJson){ + return ragSearchService.search(dataJson); + } + + @PostMapping(value = "/stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE) + public SseEmitter stream(@RequestBody String dataJson) { + + SseEmitter emitter = new SseEmitter(0L); + + modelStreamService.stream(dataJson, new ModelStreamService.StreamListener() { + + @Override + public void onMessage(String text) { + try { + emitter.send(text); + } catch (Exception e) { + emitter.completeWithError(e); + } + } + + @Override + public void onComplete() { + emitter.complete(); + } + + @Override + public void onError(Throwable t) { + emitter.completeWithError(t); + } + }); + + return emitter; + } +} \ No newline at end of file diff --git a/src/main/java/com/bw/search/entity/Constants.java b/src/main/java/com/bw/search/entity/Constants.java new file mode 100644 index 0000000..0864836 --- /dev/null +++ b/src/main/java/com/bw/search/entity/Constants.java @@ -0,0 +1,51 @@ +package com.bw.search.entity; + +/** + * 常量类 + * @author jian.mao + * @date 2025年11月6日 + * @description + */ +public class Constants { + + /****id*****/ + public static final String ID = "id"; + /***ids***/ + public static final String IDS = "ids"; + + /***检索key****/ + public static final String KEYWORD = "keyword"; + + /***知识库结果***/ + public static final String REFERENCES = "references"; + + /***问答请求参数****/ + public static final String AUTHORIZATION = "authorization"; + public static final String TEMPERATURE = "temperature"; + public static final String TOP_P = "top_p"; + public static final String MODEL = "model"; + public static final String PROMPT = "prompt"; + public static final String MASK_SENSITIVE_INFO = "mask_sensitive_info"; + public static final String MAX_TOKENS = "max_tokens"; + public static final String CHOICES = "choices"; + public static final String CONTENT = "content"; + + /****聊天体常量key****/ + public static final String MESSAGES = "messages"; + /****流****/ + public static final String STREAM = "stream"; + /***响应描述****/ + public static final String MESSAGE = "message"; + /***空常量****/ + public static final String EMPTY = ""; + /***向量结果key***/ + public static final String PROMPTRES = "promptRes"; + /***user***/ + public static final String USER = "user"; + /***角色***/ + public static final String ROLE = "role"; + /***prompt关键词标识***/ + public static final String KEYWORD_MARK = "${keyword}"; + /***prompt个人信息标识***/ + public static final String INFORMATION_MARK = "${information}"; +} diff --git a/src/main/java/com/bw/search/entity/SearchResponse.java b/src/main/java/com/bw/search/entity/SearchResponse.java new file mode 100644 index 0000000..2b51665 --- /dev/null +++ b/src/main/java/com/bw/search/entity/SearchResponse.java @@ -0,0 +1,24 @@ +package com.bw.search.entity; + +import java.util.List; + +import lombok.Data; + +@Data +public class SearchResponse { + + private String keyword; + + private List ids; + + public SearchResponse() { + } + + public SearchResponse(String keyword, List ids) { + this.keyword = keyword; + this.ids = ids; + } + + + +} diff --git a/src/main/java/com/bw/search/process/ResultSendQueue.java b/src/main/java/com/bw/search/process/ResultSendQueue.java new file mode 100644 index 0000000..4ed76be --- /dev/null +++ b/src/main/java/com/bw/search/process/ResultSendQueue.java @@ -0,0 +1,41 @@ +package com.bw.search.process; + +import java.util.Map; + +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.stereotype.Component; + +import com.alibaba.fastjson.JSONObject; +import com.bw.search.cache.ConfigCache; +import com.bw.search.entity.Constants; + +import lombok.extern.slf4j.Slf4j; + + +/** + * 接口响应结果操作类 + * @author jian.mao + * @date 2023年7月6日 + * @description + */ +@Component +@Slf4j +public class ResultSendQueue { + /** + * kafka读取结果写入队列 + * @param message + */ + @KafkaListener(topics = "#{kafkaConfig.getKafkaTopic()}") + public void consumeMessage(String message) { + // 处理接收到的消息逻辑 + try { + Map result = JSONObject.parseObject(message); + String key = result.get(Constants.ID).toString(); + ConfigCache.baseResult.put(key, result); + ConfigCache.searchResult.put(key, result); + log.info("消费知识：{}",key); + } catch (Exception e) { + log.error("结果集json转换失败，result:{},\n",message,e); + } + } +} diff --git a/src/main/java/com/bw/search/service/ModelStreamService.java b/src/main/java/com/bw/search/service/ModelStreamService.java new file mode 100644 index 0000000..0071790 --- /dev/null +++ b/src/main/java/com/bw/search/service/ModelStreamService.java @@ -0,0 +1,21 @@ +package com.bw.search.service; + +/** + * 模型流式业务层接口 + * @author jian.mao + * @date 2025年11月20日 + * @description + */ +public interface ModelStreamService { + + interface StreamListener { + void onMessage(String text); + void onComplete(); + void onError(Throwable t); + } + + /** + * 流式调用 GPT + */ + void stream(String prompt, StreamListener listener); +} \ No newline at end of file diff --git a/src/main/java/com/bw/search/service/RagSearchService.java b/src/main/java/com/bw/search/service/RagSearchService.java new file mode 100644 index 0000000..10c713b --- /dev/null +++ b/src/main/java/com/bw/search/service/RagSearchService.java @@ -0,0 +1,19 @@ +package com.bw.search.service; + +import com.bw.search.common.Res; + +/** + * rag检索业务层接口 + * @author jian.mao + * @date 2025年11月6日 + * @description + */ +public interface RagSearchService { + + /** + * 检索方法 + * @param dataJson + * @return + */ + public Res search(String dataJson); +} diff --git a/src/main/java/com/bw/search/service/impl/ModelStreamServiceImpl.java b/src/main/java/com/bw/search/service/impl/ModelStreamServiceImpl.java new file mode 100644 index 0000000..2ed4bcb --- /dev/null +++ b/src/main/java/com/bw/search/service/impl/ModelStreamServiceImpl.java @@ -0,0 +1,252 @@ +package com.bw.search.service.impl; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.bw.search.cache.ConfigCache; +import com.bw.search.config.EsConfig; +import com.bw.search.entity.Constants; +import com.bw.search.service.ModelStreamService; +import com.bw.search.utils.DateUtil; +import com.bw.search.utils.FileUtil; +import com.bw.search.utils.SpringBootKafka; + +import lombok.extern.slf4j.Slf4j; +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.BufferedSource; + + +import com.alibaba.fastjson.JSON; +import org.springframework.http.*; +import org.springframework.util.Base64Utils; +import org.springframework.web.client.RestTemplate; + +import javax.annotation.Resource; +import java.nio.charset.StandardCharsets; + + +@Service +@Slf4j +public class ModelStreamServiceImpl implements ModelStreamService { + + @Value("${model.authorization}") + private String authorization; + @Value("${model.url}") + private String modelUrl; + @Value("${model.name}") + private String modelName; + @Autowired + private SpringBootKafka springBootKafka; + @Value("${kafka.producer.topic}") + private String topic; + @Value("${prompt.path}") + private String promptPath; + + @Resource + private EsConfig esConfig; + + private final RestTemplate restTemplate = new RestTemplate(); + + private final OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(0, TimeUnit.SECONDS) // 流式接口，需要读超时为 0 + .writeTimeout(30, TimeUnit.SECONDS) + .build(); + + + @Override + public void stream(String dataJson, StreamListener listener) { + //转换对象 + JSONObject parseObject = JSONObject.parseObject(dataJson); + String id = parseObject.getString(Constants.ID); +// String keyword = parseObject.getString(Constants.KEWORD); + log.info("知识库检索开始-----"); + springBootKafka.send(topic, dataJson); + Map knowResult = getKnowledge(id); + if(knowResult == null) { + log.error("知识库获取异常！"); + return ; + } + log.info("知识库结果已获取：{}",JSONObject.toJSONString(knowResult)); + Float temperature = 0.7f; + Float topP = 0.95f; + List> messages = new ArrayList>(); + Map message = new HashMap(16); + //根据id获取源数据 + JSONObject searchRes = queryByIds(JSONObject.parseArray((String)knowResult.get(Constants.IDS), String.class)); +// log.info("数据关联结果：{}",searchRes.toJSONString()); + JSONObject hits = searchRes.getJSONObject("hits"); + //多个任务完整信息 + StringBuffer informations = new StringBuffer(); + List> hitsList = (List>) hits.get("hits"); + for (Map map : hitsList) { + Map _source = (Map) map.get("_source"); + informations.append(JSONObject.toJSONString(_source)+"\\n"); + } + String question = FileUtil.readPrompt(promptPath).replace(Constants.KEYWORD_MARK, (String)knowResult.get(Constants.KEYWORD)).replace(Constants.INFORMATION_MARK,informations.toString()) ; + message.put(Constants.CONTENT, question); + message.put(Constants.ROLE, Constants.USER); + messages.add(message); + log.info("question:{}",question);; + + Map headers = new HashMap(16); + Map params = new HashMap(16); + + params.put(Constants.MODEL, modelName); + params.put(Constants.MESSAGES, messages); + params.put(Constants.TEMPERATURE, temperature); + params.put(Constants.TOP_P, topP); + params.put(Constants.STREAM, true); + params.put(Constants.MAX_TOKENS, 8192); + + headers.put(Constants.AUTHORIZATION, authorization); + headers.put("Content-Type", "application/json"); + Request request = new Request.Builder().url(modelUrl) + .header(Constants.AUTHORIZATION, authorization).header("Content-Type", "application/json") + .post(RequestBody.create(MediaType.parse("application/json"), JSONObject.toJSONString(params))).build(); + + client.newCall(request).enqueue(new Callback() { + + @Override + public void onFailure(Call call, IOException e) { + listener.onError(e); + } + + @Override + public void onResponse(Call call, Response response) { + // 使用 try-with-resources 确保 ResponseBody 自动关闭 + try (ResponseBody body = response.body()) { + if (body == null) { + listener.onError(new IOException("Response body is null")); + return; + } + + BufferedSource source = body.source(); + while (!source.exhausted()) { + String line = source.readUtf8Line(); + if (line == null || !line.startsWith("data:")) + continue; + + if ("data: [DONE]".equals(line)) { + listener.onComplete(); + break; + } + + String json = line.substring("data:".length()).trim(); + String content = extractContent(json); + log.info("流式内容:{}",content); + if (content != null && !content.isEmpty()) { + listener.onMessage(content); + } + } + } catch (Exception e) { + listener.onError(e); + } + } + + private String extractContent(String json) { + try { + // 使用 JSON 解析，安全可靠 + JSONObject obj = JSONObject.parseObject(json); + JSONArray choices = obj.getJSONArray("choices"); + if (choices != null && !choices.isEmpty()) { + JSONObject delta = choices.getJSONObject(0).getJSONObject("delta"); + if (delta != null && delta.containsKey("content")) { + return delta.getString("content"); + } + } + } catch (Exception ignored) {} + return null; + } + }); + + } + + /** + * 获取知识库结果 + * 同步 + * @param chatId + * @return + */ + private Map getKnowledge(String id) { + Map knowResult = null; + int retryTime = 30; + while(ConfigCache.isStart) { + if(retryTime > 0) { + if(ConfigCache.baseResult.containsKey(id)) { + knowResult = (Map) ConfigCache.baseResult.get(id); + break; + }else { + log.info("知识获取中请稍后...{}",retryTime); + DateUtil.sleep(1000); + } + retryTime --; + }else { + break; + } + } + //获取之后删除 + ConfigCache.baseResult.remove(id); + return knowResult; + } + + /** + * 按 ID 批量查询 + */ + public JSONObject queryByIds(List ids) { + + // 1️⃣ 拼接 ES URL + String url = esConfig.getHost() + + "/" + esConfig.getIndex() + + "/_search"; + + // 2️⃣ 构建 terms 查询 + Map terms = new HashMap<>(); + terms.put("knowId", ids); + + Map query = new HashMap<>(); + query.put("terms", terms); + + Map body = new HashMap<>(); + body.put("query", query); + body.put("size", ids.size()); + + String jsonBody = JSON.toJSONString(body); + + // 3️⃣ 构建请求头（带 Basic Auth） + HttpHeaders headers = new HttpHeaders(); + headers.set("Content-Type", "application/json"); + + String auth = esConfig.getUsername() + ":" + esConfig.getPassword(); + String encodedAuth = Base64Utils + .encodeToString(auth.getBytes(StandardCharsets.UTF_8)); + headers.set("Authorization", "Basic " + encodedAuth); + + HttpEntity requestEntity = + new HttpEntity<>(jsonBody, headers); + + // 4️⃣ 发起请求 + ResponseEntity response = restTemplate + .postForEntity(url, requestEntity, String.class); + + // 5️⃣ 返回 JSON 结果 + return JSON.parseObject(response.getBody()); + } +} diff --git a/src/main/java/com/bw/search/service/impl/RagSearchServiceImpl.java b/src/main/java/com/bw/search/service/impl/RagSearchServiceImpl.java new file mode 100644 index 0000000..fecb062 --- /dev/null +++ b/src/main/java/com/bw/search/service/impl/RagSearchServiceImpl.java @@ -0,0 +1,68 @@ +package com.bw.search.service.impl; + +import java.util.Map; + +import org.springframework.stereotype.Service; + +import com.alibaba.fastjson.JSONObject; +import com.bw.search.cache.ConfigCache; +import com.bw.search.common.Res; +import com.bw.search.entity.Constants; +import com.bw.search.entity.SearchResponse; +import com.bw.search.service.RagSearchService; +import com.bw.search.utils.DateUtil; + +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class RagSearchServiceImpl implements RagSearchService { + + @Override + public Res search(String dataJson) { + log.info("向量检索参数：{}",dataJson); + //转换对象 + JSONObject parseObject = JSONObject.parseObject(dataJson); + String id = parseObject.getString(Constants.ID); + Map knowResult = getKnowledge(id); + if(knowResult == null) { + log.error("向量检索失败！"); + return Res.fail("知识库获取失败！"); + } + log.info("知识库结果已获取：{}",JSONObject.toJSONString(knowResult)); + //响应体数据 + SearchResponse searchResponse = new SearchResponse(); + searchResponse.setIds(JSONObject.parseArray((String)knowResult.get(Constants.IDS), String.class)); + return Res.ok(searchResponse); + } + + + + /** + * 获取知识库结果 + * 同步 + * @param chatId + * @return + */ + private Map getKnowledge(String id) { + Map knowResult = null; + int retryTime = 30; + while(ConfigCache.isStart) { + if (retryTime > 0) { + if(ConfigCache.searchResult.containsKey(id)) { + knowResult = (Map) ConfigCache.searchResult.get(id); + break; + }else { + log.info("知识获取中请稍后...{}",retryTime); + DateUtil.sleep(1000); + } + retryTime --; + }else { + break; + } + } + //获取之后删除 + ConfigCache.searchResult.remove(id); + return knowResult; + } +} diff --git a/src/main/java/com/bw/search/utils/DateUtil.java b/src/main/java/com/bw/search/utils/DateUtil.java new file mode 100644 index 0000000..5c8bef0 --- /dev/null +++ b/src/main/java/com/bw/search/utils/DateUtil.java @@ -0,0 +1,177 @@ +package com.bw.search.utils; + + +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Date; + +import lombok.extern.slf4j.Slf4j; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; + +/** + * 日期工具类 + * + * @author jian.mao + * @date 2022年11月15日 + * @description + */ +@Slf4j +public class DateUtil { + + /** + * @return + */ + public static String getTimeStrForNow() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH"); + return sdf.format(new Date()); + } + + + public static String getTimeStrForDay(long time) { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); + + return sdf.format(new Date(time * 1000)); + } + + public static String getTimeStrForDay() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); + + return sdf.format(new Date()); + } + + + public static String getDateTime() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String time = sdf.format(new Date()); + return time; + } + + public static String getDateTime(Long timestap) { + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String time = sdf.format(new Date(timestap)); + return time; + } + + public static String getDate(Long timestap) { + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + String time = sdf.format(new Date(timestap)); + return time; + } + + public static String getDateTimeForMonth() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM"); + String time = sdf.format(new Date()); + return time; + } + + /** + * 休眠 + * + * @param millis 毫秒 + */ + public static void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + /** + * 1. @Description:时间戳转时间 + * 2. @Author: ying.zhao + * 3. @Date: 2023/3/28 + */ + + public static String timestampToDate(String time) { + int thirteen = 13; + int ten = 10; + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); +// if (time.length() == thirteen) { + if (time.length() > ten) { + return sdf.format(new Date(Long.parseLong(time))); + } else { + return sdf.format(new Date(Integer.parseInt(time) * 1000L)); + } + } + + public static String parseCreated(String jsonTime){ + String formattedDateTime = getDateTime(); + try { + // 使用fastjson解析JSON数据 + JSONObject jsonObject = JSON.parseObject(jsonTime); + // 获取日期和时间的值 + JSONObject dateObject = jsonObject.getJSONObject("date"); + int day = dateObject.getIntValue("day"); + int month = dateObject.getIntValue("month"); + int year = dateObject.getIntValue("year"); + + JSONObject timeObject = jsonObject.getJSONObject("time"); + int hour = timeObject.getIntValue("hour"); + int minute = timeObject.getIntValue("minute"); + int second = timeObject.getIntValue("second"); + + // 创建LocalDateTime对象 + LocalDateTime dateTime = LocalDateTime.of(year, month, day, hour, minute, second); + + // 定义日期时间格式化器 + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + // 格式化日期时间 + formattedDateTime = dateTime.format(formatter); + } catch (Exception e) { + log.info("日期转换失败：{}",e); + } + return formattedDateTime; + } + + /** + * 字符串转换日期 + * @param format + * @param date + * @return + */ + public static Date strToDate(String format,String date){ + SimpleDateFormat sdf = new SimpleDateFormat(format); + if (date == null || date.equals("")){ + return new Date(); + }else{ + Date ru = null; + try { + ru = sdf.parse(date); + } catch (ParseException e) { + e.printStackTrace(); + } + return ru; + } + } + /** + * 日期格式话 + * @param format 日期格式 + * @param dater 要转换的日期,默认当前时间 + * @return + */ + public static String FormatDate(String format,Date date){ + String fromatDate = null; + SimpleDateFormat sdf = new SimpleDateFormat(format); + if (date == null){ + fromatDate = sdf.format(new Date()); + }else{ + fromatDate = sdf.format(date); + } + return fromatDate; + } + public static void main(String[] args) { + String time = timestampToDate("955814400000"); + System.out.println(time); + } +} diff --git a/src/main/java/com/bw/search/utils/DownLoadUtil.java b/src/main/java/com/bw/search/utils/DownLoadUtil.java new file mode 100644 index 0000000..d2f5666 --- /dev/null +++ b/src/main/java/com/bw/search/utils/DownLoadUtil.java @@ -0,0 +1,1007 @@ +package com.bw.search.utils; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.StatusLine; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.AuthCache; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.HttpClient; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.LayeredConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.BasicAuthCache; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.LaxRedirectStrategy; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.bw.search.entity.Constants; + + + + + + + + + + +/** + * 下载工具类 + * @author jian.mao + * @date 2023年9月19日 + * @description + */ +public class DownLoadUtil { + + private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"; + private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class); + /** 代理服务器(产品官网 www.16yun.cn) **/ + final static String PROXYHOST = "u270.40.tp.16yun.cn"; + final static Integer PROXYPORT = 6448; + /** 代理验证信息 **/ + final static String PROXYUSER = "16HFBVJC"; + final static String PROXYPASS = "897944"; + + private static PoolingHttpClientConnectionManager cm = null; + private static HttpRequestRetryHandler httpRequestRetryHandler = null; + private static HttpHost proxy = null; + + private static CredentialsProvider credsProvider = null; + private static RequestConfig reqConfig = null; + + static { + ConnectionSocketFactory plainsf = PlainConnectionSocketFactory + .getSocketFactory(); + LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory + .getSocketFactory(); + + Registry registry = RegistryBuilder.create().register("http", plainsf) + .register("https", sslsf).build(); + + cm = new PoolingHttpClientConnectionManager(registry); + cm.setMaxTotal(20); + cm.setDefaultMaxPerRoute(5); + + proxy = new HttpHost(PROXYHOST, PROXYPORT, "https"); + + credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(PROXYUSER, PROXYPASS)); + + reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000) + .setConnectTimeout(5000).setSocketTimeout(5000) + .setExpectContinueEnabled(false) + .setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build(); + } + + /** + * 模拟客户端get请求 + * + * @param url + * 模拟请求得url + * @param headers + * 头部信息，没有可以不传 + * @return + */ + @SafeVarargs + public static String proxyDoGet(String url, Map... headers) { + // 设置超时时间 + int timeout = 30; + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) + .setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); + AuthCache authCache = new BasicAuthCache(); + authCache.put(proxy, new BasicScheme()); + HttpClientContext localContext = HttpClientContext.create(); + localContext.setAuthCache(authCache); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + CloseableHttpClient httpClient = httpBuilder + .setDefaultSocketConfig(socketConfig) + .setDefaultRequestConfig(config) + .setDefaultCredentialsProvider(credsProvider).build(); + HttpGet httpGet = new HttpGet(url); + httpGet.setConfig(reqConfig); + if (headers != null && headers.length > 0) { + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key, tempHeaders.get(key).toString()); + } + } else { + httpGet.setHeader("Accept", + "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + } + CloseableHttpResponse response = null; + String html = ""; + int notFundCode = 404; + int successCode = 200; + try { + response = httpClient.execute(httpGet, localContext); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + if (statusLine.getStatusCode() == successCode) { + if (responseEntity != null) { + html = EntityUtils.toString(responseEntity, "utf-8"); + System.out.println("响应内容长度为:" + + responseEntity.getContentLength()); + // 下载结果为空不正常 + if (html.equals(Constants.EMPTY)) { + html = "Download failed error is:reslut is null"; + } + } + } else if (statusLine.getStatusCode() == notFundCode) { + html = "

页面404，正常结束请求即可

"; + } else { + throw new Exception("请求错误，code码为：" + statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:reslut is null"; + }finally{ + try { + response.close(); + httpClient.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + return html; + + } + + + public static String httpsslProxyGet(String url, Map... headers) throws Exception { + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connManager.setMaxTotal(50); + connManager.setDefaultMaxPerRoute(10); + HttpClients.custom().setConnectionManager(connManager); + // 设置超时时间 + int timeout = 30; + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) + .setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); + AuthCache authCache = new BasicAuthCache(); + authCache.put(proxy, new BasicScheme()); + HttpClientContext localContext = HttpClientContext.create(); + localContext.setAuthCache(authCache); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + CloseableHttpClient httpClient = httpBuilder + .setConnectionManager(connManager) + .setDefaultSocketConfig(socketConfig) + .setDefaultRequestConfig(config) + .setDefaultCredentialsProvider(credsProvider).build(); + HttpGet httpGet = new HttpGet(url); + httpGet.setConfig(reqConfig); + if (headers != null && headers.length > 0) { + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key, tempHeaders.get(key).toString()); + } + } else { + httpGet.setHeader("Accept", + "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + } + CloseableHttpResponse response = null; + String html = ""; + int notFundCode = 404; + int successCode = 200; + try { + response = httpClient.execute(httpGet, localContext); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + if (statusLine.getStatusCode() == successCode) { + if (responseEntity != null) { + html = EntityUtils.toString(responseEntity, "utf-8"); + System.out.println("响应内容长度为:" + + responseEntity.getContentLength()); + // 下载结果为空不正常 + if (html.equals(Constants.EMPTY)) { + html = "Download failed error is:reslut is null"; + } + } + } else if (statusLine.getStatusCode() == notFundCode) { + html = "

页面404，正常结束请求即可

"; + } else { + throw new Exception("请求错误，code码为：" + statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:reslut is null"; + }finally{ + try { + response.close(); + httpClient.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + return html; + + } + + + /** + * json参数方式POST提交 + * @param url + * @param params + * @return + */ + public static String doPost(String url, String params, Map... headers){ + String strResult = ""; + //设置超时时间 + int timeout = 600; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); +// AuthCache authCache = new BasicAuthCache(); +// authCache.put(proxy, new BasicScheme()); +// HttpClientContext localContext = HttpClientContext.create(); +// localContext.setAuthCache(authCache); + // 1. 获取默认的client实例 + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); + HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); +// HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) +// .setDefaultCredentialsProvider(credsProvider).build(); + // 2. 创建httppost实例 + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + if (headers != null && headers.length > 0) { + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key, tempHeaders.get(key).toString()); + } + } else { + httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); + } + HttpResponse resp = null; + try { + httpPost.setEntity(new StringEntity(params,"utf-8")); + resp = client.execute(httpPost); +// resp = client.execute(httpPost,localContext); + StatusLine statusLine = resp.getStatusLine(); + System.out.println("响应状态为:" + resp.getStatusLine()); + int notFundCode = 404; + int successCode = 200; + if(statusLine.getStatusCode() == successCode){ + // 7. 获取响应entity + HttpEntity respEntity = resp.getEntity(); + strResult = EntityUtils.toString(respEntity, "UTF-8"); + if(strResult.equals(Constants.EMPTY)){ + strResult = "Download failed error is:reslut is null"; + } + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + return strResult; + } + public static String httpPost(String url,String params) { + String html=""; + html = doPost(url,params); + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(5000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + html = doPost(url,params); + } + return html; + } + /** + * 绕过验证 + * + * @return + * @throws NoSuchAlgorithmException + * @throws KeyManagementException + */ + public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException { + SSLContext sc = SSLContext.getInstance("SSLv3"); + + // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法 + X509TrustManager trustManager = new X509TrustManager() { + @Override + public void checkClientTrusted( + java.security.cert.X509Certificate[] paramArrayOfX509Certificate, + String paramString) throws CertificateException { + } + + @Override + public void checkServerTrusted( + java.security.cert.X509Certificate[] paramArrayOfX509Certificate, + String paramString) throws CertificateException { + } + + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + }; + + sc.init(null, new TrustManager[] { trustManager }, null); + return sc; + } + /** + * 模拟请求 + * + * @param url 资源地址 + * @param map 参数列表 + * @param encoding 编码 + * @return + * @throws NoSuchAlgorithmException + * @throws KeyManagementException + * @throws IOException + * @throws ClientProtocolException + */ + public static String httpsslGet(String url,Map ... headers) { + String html=""; + CloseableHttpClient client = null; + HttpEntity responseEntity = null; + CloseableHttpResponse response = null; + try { + log.debug("DownLoadUtil------------->设置下载相关信息， start...."); + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connManager.setMaxTotal(50); + connManager.setDefaultMaxPerRoute(10); + HttpClients.custom().setConnectionManager(connManager); + //设置超时时间 + int timeout = 30; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + // 设置重定向策略 + LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy(); + //创建自定义的httpclient对象 + client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build(); +// CloseableHttpClient client = HttpClients.createDefault(); + + HttpGet httpGet = new HttpGet(url); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); + } + log.debug("DownLoadUtil------------->设置下载相关信息， end...."); + try { + int notFundCode = 404; + int successCode = 200; + log.debug("DownLoadUtil------------->下载执行，start...."); + httpGet.setConfig(config); + response = client.execute(httpGet); + log.debug("DownLoadUtil------------->下载执行，end...."); + // 从响应模型中获取响应实体 + StatusLine statusLine = response.getStatusLine(); + log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is start ...."); + System.out.println("响应状态为:" + response.getStatusLine()); + responseEntity = response.getEntity(); + log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is end ...."); + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + System.out.println("响应内容长度为:" + responseEntity.getContentLength()); + } + }else if(statusLine.getStatusCode() == notFundCode){ + html = "

页面404，正常结束请求即可

"; + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + }finally{ + try { + responseEntity.getContent().close(); + response.close(); + client.close(); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + return html; + } + + public static String httpSSLGet(String url,Map ... headers) { + String html=""; + html = httpsslGet(url,headers); + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(30000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + html = httpsslGet(url,headers); + } + return html; + } + public static String doPostFrom(String url,Map param,Map ... headers){ + //设置超时时间 + int timeout = 15; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); +// AuthCache authCache = new BasicAuthCache(); +// authCache.put(proxy, new BasicScheme()); +// HttpClientContext localContext = HttpClientContext.create(); +// localContext.setAuthCache(authCache); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); +// HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) +// .setDefaultCredentialsProvider(credsProvider).build(); + HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"); + httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); + httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"); +// httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html"); + } + // 创建请求参数 + List list = new LinkedList<>(); + for (String key : param.keySet()) { + BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); + list.add(param1); + } + // 使用URL实体转换工具 + String html=""; + try { + UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); + httpPost.setEntity(entityParam); + HttpResponse response = httpClient.execute(httpPost); +// HttpResponse response = httpClient.execute(httpPost,localContext); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + int notFundCode = 404; + int successCode = 200; + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + } + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + + return html; + + } + public static String httpPostForm(String url,Map params,Map ... headers) { + String html=""; + html = doPostFrom(url,params); + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(5000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + html = doPostFrom(url,params,headers); + } + return html; + } + + public static String dosslPost(String url,String params,Map ... headers) { + String html=""; + CloseableHttpClient client = null; + HttpEntity responseEntity = null; + CloseableHttpResponse response = null; + try { + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + HttpClients.custom().setConnectionManager(connManager); + //设置超时时间 + int timeout = 5; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + //创建自定义的httpclient对象 + client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); +// CloseableHttpClient client = HttpClients.createDefault(); + // 2. 创建httppost实例 + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); + } + + try { + httpPost.setEntity(new StringEntity(params,"utf-8")); + response = client.execute(httpPost); + int notFundCode = 404; + int successCode = 200; + // 从响应模型中获取响应实体 + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + responseEntity = response.getEntity(); + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + System.out.println("响应内容长度为:" + responseEntity.getContentLength()); + } + }else if(statusLine.getStatusCode() == notFundCode){ + html = "

页面404，正常结束请求即可

"; + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + }finally{ + try { + responseEntity.getContent().close(); + response.close(); + client.close(); + } catch (UnsupportedOperationException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + + return html; + } + public static String dosslPostForm(String url,Map param,Map ... headers) { + String html=""; + try { + //采用绕过验证的方式处理https请求 + SSLContext sslcontext = createIgnoreVerifySSL(); + + // 设置协议http和https对应的处理socket链接工厂的对象 + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslcontext)) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + HttpClients.custom().setConnectionManager(connManager); + //设置超时时间 + int timeout = 5; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + //创建自定义的httpclient对象 + CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); +// CloseableHttpClient client = HttpClients.createDefault(); + // 2. 创建httppost实例 + HttpPost httpPost = new HttpPost(url); +// httpPost.setConfig(reqConfig); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpPost.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); + httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); + } + + // 创建请求参数 + List list = new LinkedList<>(); + for (String key : param.keySet()) { + BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); + list.add(param1); + } + // 使用URL实体转换工具 + try { + UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); + httpPost.setEntity(entityParam); + HttpResponse response = client.execute(httpPost); +// HttpResponse response = httpClient.execute(httpPost,localContext); + // 从响应模型中获取响应实体 + int notFundCode = 404; + int successCode = 200; + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + } + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + + + return html; + } + public static String httpSSLPostForm(String url,Map params,Map ...headers) { + String html=""; + try { + html = dosslPostForm(url,params,headers); + } catch (Exception e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(30000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + try { + html = dosslPostForm(url,params,headers); + } catch (Exception e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + } + return html; + } + public static String httpSSLPost(String url,String params,Map ...headers) { + String html=""; + try { + html = dosslPost(url,params,headers); + } catch (Throwable e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + int i = 1; + while(true){ + if(html.contains("Download failed error is:")){ + log.error("DownLoadUtil------------->download is failure,url is:"+url); + DateUtil.sleep(30000); + i++; + }else{ + break; + } + if(i > 5){ + break; + } + try { + html = dosslPost(url,params,headers); + } catch (Throwable e) { + e.printStackTrace(); + // TODO: handle exception + html = "Download failed error is:Exception!"; + } + } + return html; + } + + /** + * 模拟客户端get请求 + * @param url 模拟请求得url + * @param headers 头部信息，没有可以不传 + * @return + */ + public static String doGet(String url,Map ... headers){ + //设置超时时间 + int timeout = 15; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(10000) + .setTcpNoDelay(true).build(); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); + HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); + HttpGet httpGet = new HttpGet(url); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + } + String html=""; + try { + int notFundCode = 404; + int successCode = 200; + HttpResponse response = httpClient.execute(httpGet); + // 从响应模型中获取响应实体 + HttpEntity responseEntity = response.getEntity(); + StatusLine statusLine = response.getStatusLine(); + System.out.println("响应状态为:" + response.getStatusLine()); + if(statusLine.getStatusCode() == successCode){ + if (responseEntity != null) { + html=EntityUtils.toString(responseEntity,"utf-8"); + if(html.equals("")){ + html = "Download failed error is:reslut is null"; + } + } + }else if(statusLine.getStatusCode() == notFundCode){ + html = "

页面404，正常结束请求即可

"; + }else{ + throw new Exception("请求错误，code码为："+statusLine.getStatusCode()); + } + } catch (Exception e) { + e.printStackTrace(); + html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); + } + return html; + + } + + /** + * 文件下载 + * @param fileURL 文件连接 + * @param saveFilePath 文件存储地址 + * @throws IOException + */ + public static void downloadFile(String fileURL, String saveFilePath,Map ... headers) { + //设置超时时间 + int timeout = 60; + RequestConfig config = RequestConfig.custom(). + setConnectTimeout(timeout * 1000). + setConnectionRequestTimeout(timeout * 1000). + setSocketTimeout(timeout * 1000).build(); + SocketConfig socketConfig = SocketConfig.custom() + .setSoKeepAlive(false) + .setSoLinger(1) + .setSoReuseAddress(true) + .setSoTimeout(timeout * 1000) + .setTcpNoDelay(true).build(); + HttpClientBuilder httpBuilder = HttpClientBuilder.create(); + httpBuilder.setUserAgent(ua); + HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); + HttpGet httpGet = new HttpGet(fileURL); + if(headers != null && headers.length > 0){ + Map tempHeaders = headers[0]; + for (String key : tempHeaders.keySet()) { + httpGet.setHeader(key,tempHeaders.get(key).toString()); + } + }else{ + httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); + } + try { + int successCode = 200; + HttpResponse response = httpClient.execute(httpGet); + // 从响应模型中获取响应实体 + StatusLine statusLine = response.getStatusLine(); + int statusCode = statusLine.getStatusCode(); + if(statusCode == successCode){ + // 获取响应体 + HttpEntity entity = response.getEntity(); + if (entity != null) { + // 创建输入流来读取文件内容 + try (InputStream inputStream = entity.getContent(); + FileOutputStream fileOutputStream = new FileOutputStream(new File(saveFilePath))) { + + byte[] buffer = new byte[4096]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + fileOutputStream.write(buffer, 0, bytesRead); + } + } + log.info("文件下载成功，保存路径为: {}" , saveFilePath); + } + } else { + log.error("文件下载失败，HTTP 响应状态码为: {}" , statusCode); + } + EntityUtils.consume(response.getEntity()); + } catch (Exception e) { + log.error("下载文件异常：{}",e); + } + + } + + /** + * 删除文件 + * @param filePath + */ + public static void delFile(String filePath) { + try { + // 创建 File 对象 + File file = new File(filePath); + // 检查文件是否存在 + if (file.exists()) { + // 尝试删除文件 + if (file.delete()) { + log.info("文件删除成功: " + filePath); + } else { + log.error("无法删除文件: " + filePath); + } + } else { + log.warn("文件不存在: " + filePath); + } + } catch (Exception e) { + // TODO: handle exception + log.error("删除文件异常：{}",filePath); + } + } + +} diff --git a/src/main/java/com/bw/search/utils/FileUtil.java b/src/main/java/com/bw/search/utils/FileUtil.java new file mode 100644 index 0000000..3cb1b89 --- /dev/null +++ b/src/main/java/com/bw/search/utils/FileUtil.java @@ -0,0 +1,60 @@ +package com.bw.search.utils; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingDeque; + +import org.apache.commons.io.FileUtils; + +import com.alibaba.fastjson.JSONObject; + +/** + * 文件工具类 + * @author jian.mao + * @date 2023年7月14日 + * @description + */ +public class FileUtil { + + /** + * 数据写入文件 + * @param Path 文件路径 + * @param result 数据 + * @throws IOException + */ + public static void writeFile(String path,String result){ + try { + FileWriter fw = new FileWriter(path,true); + fw.write(result+"\n"); + fw.flush(); + fw.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 读取prompt + * @param path + * @return + */ + public static String readPrompt(String path){ + StringBuffer prompt = new StringBuffer(); + File file = new File(path); + if(file.exists()){ + List tasks = null; + try { + tasks = FileUtils.readLines(file,"UTF-8"); + } catch (IOException e) { + e.printStackTrace(); + } + for (String taskStr : tasks) { + prompt.append(taskStr); + } + } + return prompt.toString(); + } +} diff --git a/src/main/java/com/bw/search/utils/SpringBootKafka.java b/src/main/java/com/bw/search/utils/SpringBootKafka.java new file mode 100644 index 0000000..5e48425 --- /dev/null +++ b/src/main/java/com/bw/search/utils/SpringBootKafka.java @@ -0,0 +1,45 @@ +package com.bw.search.utils; + +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.SendResult; +import org.springframework.stereotype.Component; +import org.springframework.util.concurrent.ListenableFuture; +import org.springframework.util.concurrent.ListenableFutureCallback; + +/** + * @DESCRIPTION:SpringBootKafka 工具类 + */ +@Slf4j +@Component +public class SpringBootKafka { + @Autowired + private KafkaTemplate kafkaTemplate; + /** + * 自定义topicKafkaTemplate + */ + /** + * public static final String TOPIC = "companyBussTest"; + **/ + public void send(String topic, String message) { + String obj2String = JSONObject.toJSONString(message); + log.info("准备发送消息为：{}", obj2String); + //发送消息 + ListenableFuture> future = kafkaTemplate.send(topic, message); + future.addCallback(new ListenableFutureCallback>() { + @Override + public void onFailure(Throwable throwable) { + //发送失败的处理 + log.info(topic + " - 生产者 发送消息失败：" + throwable.getMessage()); + } + + @Override + public void onSuccess(SendResult stringObjectSendResult) { + //成功的处理 + log.info(topic + " - 生产者 发送消息成功：" + stringObjectSendResult.toString()); + } + }); + } +} diff --git a/src/main/java/com/bw/search/utils/ThrowMessageUtil.java b/src/main/java/com/bw/search/utils/ThrowMessageUtil.java new file mode 100644 index 0000000..4cb3cc5 --- /dev/null +++ b/src/main/java/com/bw/search/utils/ThrowMessageUtil.java @@ -0,0 +1,23 @@ +package com.bw.search.utils; + +import java.io.PrintWriter; +import java.io.StringWriter; + +/** + * @author jian.mao + * @date 2023年3月22日 + * @description + */ +public class ThrowMessageUtil { + + /** + * 获取异常信息 + * @param t + * @return + */ + public static String getErrmessage(Throwable t){ + StringWriter stringWriter=new StringWriter(); + t.printStackTrace(new PrintWriter(stringWriter,true)); + return stringWriter.getBuffer().toString(); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..b1e5705 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,73 @@ +logging: + level: + root: info + path: ../logs +server: + port: 8001 + servlet: + context-path: /nvps + tomcat: + uri-encoding: utf-8 + max-connections: 20000 + max-http-form-post-size: 1 + max-threads: 1000 +model: + url: https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions + authorization: Bearer sk-c5f56c1c8a6447b3a6c646a3f14085c4 + name: qwen-max-latest +spring: + kafka: + bootstrap-servers: node-01:19092,node-02:19092,node-03:19092 + producer: + retries: 0 + #当有多个消息需要被发送到同一个分区时，生产者会把它们放在同一个批次里。该参数指定了一个批次可以使用的内存大小，按照字节数计算。 + batch-size: 16384 + # 设置生产者内存缓冲区的大小。 + buffer-memory: 33554432 + # 键的序列化方式 + key-serializer: org.apache.kafka.common.serialization.StringSerializer + # 值的序列化方式 + value-serializer: org.apache.kafka.common.serialization.StringSerializer + # acks=0 ： 生产者在成功写入消息之前不会等待任何来自服务器的响应。 + # acks=1 ： 只要集群的首领节点收到消息，生产者就会收到一个来自服务器成功响应。 + # acks=all ：只有当所有参与复制的节点全部收到消息时，生产者才会收到一个来自服务器的成功响应。 + acks: 1 + consumer: + # 自动提交的时间间隔 在spring boot 2.X 版本中这里采用的是值的类型为Duration 需要符合特定的格式，如1S,1M,2H,5D + auto-commit-interval: 1S + # 该属性指定了消费者在读取一个没有偏移量的分区或者偏移量无效的情况下该作何处理： + # latest（默认值）在偏移量无效的情况下，消费者将从最新的记录开始读取数据（在消费者启动之后生成的记录） + # earliest ：在偏移量无效的情况下，消费者将从起始位置读取分区的记录 + auto-offset-reset: earliest + # 是否自动提交偏移量，默认值是true,为了避免出现重复数据和数据丢失，可以把它设置为false,然后手动提交偏移量 + enable-auto-commit: true + # 键的反序列化方式 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + # 值的反序列化方式 + value-deserializer: org.apache.kafka.common.serialization.StringDeserializer + #消费组 + group-id: test4 + #消费者并发线程数 + concurrency: 4 + #超时时间 + max-poll-interval-ms: 60000 + #listener: + # 在侦听器容器中运行的线程数。 + #concurrency: 5 + #listner负责ack，每调用一次，就立即commit + #ack-mode: manual_immediate + #missing-topics-fatal: false +es: + host: http://127.0.0.1:9200 + username: elastic + password: es@123 + index: nvps_person_info + +kafka: + consumer: + #自定义topic + topic: nvps_search_data + producer: + topic: nvps_know_base +prompt: + path: ./prompt \ No newline at end of file diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..87c77c7 --- /dev/null +++ b/src/main/resources/logback-spring.xml @@ -0,0 +1,36 @@ + + + + + + + + + true + + ${logging.level} + + + ${logging.path}/ragEngineInfo.log + + + ${logging.path}/ragEngineInfo.log.%d{yyyy-MM-dd} + 7 + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n + UTF-8 + + + + + + + + diff --git a/src/test/java/com/bw/AppTest.java b/src/test/java/com/bw/AppTest.java new file mode 100644 index 0000000..fc367fc --- /dev/null +++ b/src/test/java/com/bw/AppTest.java @@ -0,0 +1,38 @@ +package com.bw; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } + + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } +} diff --git a/target/.gitignore b/target/.gitignore new file mode 100644 index 0000000..73ef6b1 --- /dev/null +++ b/target/.gitignore @@ -0,0 +1,4 @@ +/classes/ +/test-classes/ +/logs/ +/target/ \ No newline at end of file