Browse Source

代理ip管理服务

master
maojian 6 months ago
commit
4ec92f513f
  1. 40
      .classpath
  2. 3
      .gitignore
  3. 23
      .project
  4. 5
      .settings/org.eclipse.core.resources.prefs
  5. 9
      .settings/org.eclipse.jdt.core.prefs
  6. 4
      .settings/org.eclipse.m2e.core.prefs
  7. 2
      ipPool_Manger.iml
  8. 155
      pom.xml
  9. 27
      src/main/java/com/bfd/crawl/Application.java
  10. 162
      src/main/java/com/bfd/crawl/controller/GetDataController.java
  11. 22
      src/main/java/com/bfd/crawl/model/Ips.java
  12. 54
      src/main/java/com/bfd/crawl/process/ipManger.java
  13. 69
      src/main/java/com/bfd/crawl/util/SendEmail.java
  14. 32
      src/main/resources/application.yml
  15. 37
      src/main/resources/logback-spring.xml

40
.classpath

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="optional" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
<attribute name="optional" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

3
.gitignore

@ -0,0 +1,3 @@
/target/
/logs/
/.idea/

23
.project

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>ipPool_Manger</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>

5
.settings/org.eclipse.core.resources.prefs

@ -0,0 +1,5 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding/<project>=UTF-8

9
.settings/org.eclipse.jdt.core.prefs

@ -0,0 +1,9 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.methodParameters=generate
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8

4
.settings/org.eclipse.m2e.core.prefs

@ -0,0 +1,4 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1

2
ipPool_Manger.iml

@ -0,0 +1,2 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4" />

155
pom.xml

@ -0,0 +1,155 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.4.RELEASE</version>
</parent>
<groupId>org.example</groupId>
<artifactId>ipPool_Manger</artifactId>
<version>1.0-SNAPSHOT</version>
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>3.9.1</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.10</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>2.0.17</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.30</version>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>2.0.0</version>
</dependency>
<!--发邮件-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-mail</artifactId>
</dependency>
</dependencies>
<build>
<!-- <pluginManagement> --><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
<plugin>
<artifactId>maven-site-plugin</artifactId>
<version>3.7.1</version>
</plugin>
<plugin>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<!-- spring-boot-maven-plugin插件就是打包spring boot应用的 -->
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<mainClass>com.bfd.crawl.Application</mainClass>
<layout>ZIP</layout>
<includes>
<include>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
</include>
</includes>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<type>jar</type>
<includeTypes>jar</includeTypes>
<includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/libs</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<!-- </pluginManagement> -->
</build>
</project>

27
src/main/java/com/bfd/crawl/Application.java

@ -0,0 +1,27 @@
package com.bfd.crawl;//import com.bfd.crawl.process.ElasticsearchMaoJian;
import com.bfd.crawl.process.ipManger;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
/**
* @author:zhaoying
* @className:Application
* @version:1.0
* @description:主入口
* @Date:2023-12-15 14:30:41
*/
@SpringBootApplication
@EnableScheduling
@Slf4j
public class Application {
public static void main(String[] args) {
SpringApplication.run(com.bfd.crawl.Application.class, args);
ipManger ipManger = new ipManger();
Thread ipMangerThread = new Thread(ipManger);
ipMangerThread.start();
}
}

162
src/main/java/com/bfd/crawl/controller/GetDataController.java

@ -0,0 +1,162 @@
package com.bfd.crawl.controller;
import com.bfd.crawl.model.Ips;
import com.bfd.crawl.util.SendEmail;
import lombok.extern.slf4j.Slf4j;
import okhttp3.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* @PROJECT_NAME:
* @DESCRIPTION:获取一个时效内的ip
* @AUTHOR: ying.zhao
* @DATE: 2023/4/3 18:17
*/
@Slf4j
@RestController
public class GetDataController {
@Autowired
private SendEmail sendEmail;
private static final Lock lock = new ReentrantLock();
@GetMapping(value = "/getIp")
public String testFrom() {
// 静态锁所有请求共享
String ip = "";
String html = "";
// while (ip.equals("")) {
Long randomKey = getRandomEligibleKey();
if (randomKey != null) {
log.info("时效内的key: {}", randomKey);
ip = Ips.ipMap.get(randomKey);
log.info("时效内的ip: {}", ip);
} else {
if (lock.tryLock()) {
try {
// 执行你的业务逻辑
if (Ips.ipMap.size() < 1) {
html = getIpList();
if(html.contains("msg")){
log.info("ip请求异常");
return html;
}
} else {
log.info("已获取一次iP");
}
} finally {
// 确保在执行完后释放锁
lock.unlock();
}
} else {
log.info("锁上了 等会儿吧。。。。。。。。。");
}
}
return ip;
}
/**
* @Param:
* @DESCRIPTION: 请求获取ip
* @Author: ying.zhao
* @date: 2024/9/12
*/
public String getIpList() {
OkHttpClient.Builder builder = new OkHttpClient.Builder();
builder.readTimeout(200, TimeUnit.SECONDS);
builder.connectTimeout(200, TimeUnit.SECONDS);
OkHttpClient client = new OkHttpClient().newBuilder()
.build();
MediaType mediaType = MediaType.parse("text/plain");
RequestBody body = RequestBody.create(mediaType, "");
Request request = new Request.Builder()
.url("http://api.tianqiip.com/getip?secret=x7it4ebj&num=10&type=txt&port=1&mr=1&sign=899ce225ae2cd14731d60fe48c338d0a")
.method("GET", null)
.build();
Response response = null;
String html = "";
try {
response = client.newCall(request).execute();
html = response.body().string();
} catch (IOException e) {
e.printStackTrace();
}
if (!html.equals("")) {
log.info("请求得到页面:{}", html);
if (html.contains("msg")) {
try {
sendEmail.send("Alarm", "ip请求异常报警!!!!!!!!", html);
} catch (Exception e) {
e.printStackTrace();
}
} else {
// 按换行符拆分字符串
Ips.ipList = new ArrayList<String>(Arrays.asList(html.split("\n")));
for (String ip : Ips.ipList) {
log.info("ip:{}", ip);
Ips.ipMap.put(System.currentTimeMillis(), ip);
try {
Thread.sleep(1); // 延迟1毫秒
} catch (InterruptedException e) {
Thread.currentThread().interrupt(); // 处理中断异常
}
}
}
}
return html;
}
/**
* @Param:
* @DESCRIPTION: 随机获取一个eligibleKeys中的key确保每轮都被取到一次
* @Author: ying.zhao
* @date: 2024/9/12
*/
public static Long getRandomEligibleKey() {
if (Ips.eligibleKeys.isEmpty()) {
return null; // 如果列表为空返回null
}
// 如果所有符合条件的key都已经被取用重新填充轮次
if (Ips.keysInCurrentRound.size() == Ips.eligibleKeys.size()) {
Ips.keysInCurrentRound.clear(); // 清空当前轮次的标记
}
// 筛选出尚未被取用的key
List<Long> remainingKeys = new ArrayList<>();
for (Long key : Ips.eligibleKeys) {
if (!Ips.keysInCurrentRound.contains(key)) {
remainingKeys.add(key);
}
}
// 如果没有剩余的key可供选择
if (remainingKeys.isEmpty()) {
return null; // 所有key都已被取用
}
// 随机选择一个尚未被取用的key
Random random = new Random();
int randomIndex = random.nextInt(remainingKeys.size());
Long randomKey = remainingKeys.get(randomIndex);
// 标记已取用的key
Ips.keysInCurrentRound.add(randomKey);
return randomKey;
}
}

22
src/main/java/com/bfd/crawl/model/Ips.java

@ -0,0 +1,22 @@
package com.bfd.crawl.model;
import lombok.Data;
import java.util.*;
import java.util.logging.Handler;
/**
* @PROJECT_NAME:
* @DESCRIPTION:
* @AUTHOR: ying.zhao
* @DATE: 2023/4/4 9:39
*/
@Data
public class Ips {
public static List<String> ipList = new ArrayList<>();
public static Map<Long,String> ipMap = new HashMap<Long,String>(16);
// 临时列表存储符合条件的key
public static List<Long> eligibleKeys = new ArrayList<>();
// 用于标记当前轮次中已取用的key
public static Set<Long> keysInCurrentRound = Collections.synchronizedSet(new HashSet<>());
}

54
src/main/java/com/bfd/crawl/process/ipManger.java

@ -0,0 +1,54 @@
package com.bfd.crawl.process;
import com.bfd.crawl.model.Ips;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.stream.Collectors;
/**
* @PROJECT_NAME: ipPool_Manger
* @DESCRIPTION:
* @AUTHOR: ying.zhao
* @DATE: 2024/9/12 14:22
*/
@Slf4j
public class ipManger implements Runnable {
@Override
public void run() {
// long tenMinutesInMillis = 3 * 60 * 1000; // 10分钟对应的毫秒数
long tenMinutesInMillis =(long) (2.7 * 60 * 1000); // 2.7分钟对应的毫秒数
while (true) {
log.info("正在监控ipMap中一共有:{}个", Ips.ipMap.size());
log.info("正在监控可用ip有:{}个", Ips.eligibleKeys.size());
long currentTime = System.currentTimeMillis(); // 当前时间戳
if (Ips.ipMap.size() > 0) {
// 添加符合条件的key
for (Long key : Ips.ipMap.keySet()) {
if (currentTime - key <= tenMinutesInMillis && !Ips.eligibleKeys.contains(key)) {
Ips.eligibleKeys.add(key); // 只添加还未加入的key
}
}
// 找到超过10分钟的key先做入库操作
// List<Long> keysToBeRemoved = Ips.eligibleKeys.stream()
// .filter(key -> currentTime - key > tenMinutesInMillis)
// .collect(Collectors.toList());
// if (!keysToBeRemoved.isEmpty()) {
//// saveToDatabase(keysToBeRemoved); // 保存这些key到数据库
// }
// 移除超过10分钟的key
Ips.eligibleKeys.removeIf(key -> currentTime - key > tenMinutesInMillis);
Ips.ipMap.entrySet().removeIf(entry -> currentTime - entry.getKey() > tenMinutesInMillis);
}
// 休眠一段时间避免频繁循环
try {
Thread.sleep(2 * 1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}

69
src/main/java/com/bfd/crawl/util/SendEmail.java

@ -0,0 +1,69 @@
package com.bfd.crawl.util;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import java.io.UnsupportedEncodingException;
import java.util.Date;
import java.util.Properties;
/**
* @author:zhaoying
* @className:SendEmail
* @version:1.0
* @description:
* @Date:2023-08-03 18:19:41
*/
@Slf4j
@Component
public class SendEmail {
@Value("${spring.email.host}")
private String host;
@Value("${spring.email.auth}")
private String auth;
@Value("${spring.email.user}")
private String user;
@Value("${spring.email.password}")
private String password;
@Value("${spring.email.receive}")
private String receive;
public void send(String sendUser, String title, String msage) throws MessagingException, UnsupportedEncodingException {
Properties props = new Properties();
props.put("mail.smtp.host",host);
props.put("mail.smtp.auth", auth);
Session session = Session.getDefaultInstance(props, null);
Transport transport = session.getTransport();
transport.connect(user, password);
MimeMessage msg = new MimeMessage(session);
msg.setSentDate(new Date());
//邮件发送人
InternetAddress fromAddress = new InternetAddress(user, sendUser);
msg.setFrom(fromAddress);
//邮件接收人
String[] receives = receive.split("#@#");
for (int i=0;i<receives.length;i++) {
InternetAddress[] toAddress = new InternetAddress[]{new InternetAddress(receives[i])};
msg.setRecipients(Message.RecipientType.TO, toAddress);
//邮件主题
msg.setSubject(title, "UTF-8");
//邮件内容和格式
msg.setContent(msage, "text/html;charset=UTF-8");
msg.saveChanges();
//发送
transport.sendMessage(msg, msg.getAllRecipients());
}
}
public static void main(String[] args) throws Exception {
}
}

32
src/main/resources/application.yml

@ -0,0 +1,32 @@
server:
port: 5476
tomcat:
uri-encoding: utf-8
max-connections: 20000
max-http-form-post-size: -1
max-threads: 1000
# servlet:
# context-path: /test
logging:
level:
root: info
path: ./logs
spring:
main:
web-application-type: servlet
datasource:
url: jdbc:mysql://172.18.1.102:3306/zhiwang?serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8&useSSL=true
username: crawl
password: crawl123!@#
driver-class-name: com.mysql.cj.jdbc.Driver
email:
host: intmail.baifendian.com
auth: true
user: bfd_crawler_alarm@baifendian.com
password: z26Iyf3vMRb5ejrI
receive: ying.zhao@percent.cn#@#huanhuan.shi@percent.cn#@#jian.mao@percent.cn
# receive: ying.zhao@percent.cn
mybatis:
type-aliases-package: com.bfd.crawl.model

37
src/main/resources/logback-spring.xml

@ -0,0 +1,37 @@
<configuration>
<!-- 属性文件:在properties文件中找到对应的配置项 -->
<springProperty scope="context" name="logging.path" source="logging.path"/>
<springProperty scope="context" name="logging.level" source="logging.level.com.bfd"/>
<!-- 默认的控制台日志输出,一般生产环境都是后台启动,这个没太大作用 -->
<appender name="STDOUT"
class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<Pattern>%d{HH:mm:ss.SSS} %-5level %logger{80} - %msg%n</Pattern>
</encoder>
</appender>
<appender name="GLMAPPER-LOGGERONE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<append>true</append>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>${logging.level}</level>
</filter>
<file>
<!-- 日志名 -->
${logging.path}/data.log
</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${logging.path}/data.log.%d{yyyy-MM-dd}</FileNamePattern>
<MaxHistory>7</MaxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<root level="info">
<appender-ref ref="GLMAPPER-LOGGERONE"/>
<appender-ref ref="STDOUT"/>
</root>
</configuration>
Loading…
Cancel
Save