commit 4ec92f513fd21fcac77813ab86044e5c81c1a3a0 Author: maojian <550076202@qq.com> Date: Wed Jan 8 15:18:21 2025 +0800 代理ip管理服务 diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..f7e4a1d --- /dev/null +++ b/.classpath @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e3b721c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/target/ +/logs/ +/.idea/ \ No newline at end of file diff --git a/.project b/.project new file mode 100644 index 0000000..28f0888 --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + ipPool_Manger + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..839d647 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..71df522 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,9 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.methodParameters=generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/ipPool_Manger.iml b/ipPool_Manger.iml new file mode 100644 index 0000000..78b2cc5 --- /dev/null +++ b/ipPool_Manger.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..16f361c --- /dev/null +++ b/pom.xml @@ -0,0 +1,155 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.2.4.RELEASE + + org.example + ipPool_Manger + 1.0-SNAPSHOT + http://www.example.com + + UTF-8 + 1.8 + 1.8 + + + + junit + junit + 4.11 + test + + + org.springframework.boot + spring-boot-starter-web + + + com.squareup.okhttp3 + okhttp + 3.9.1 + + + org.projectlombok + lombok + 1.18.10 + compile + + + com.alibaba + fastjson + 2.0.17 + + + org.apache.httpcomponents + httpclient + 4.5.3 + + + mysql + mysql-connector-java + 8.0.30 + + + org.mybatis.spring.boot + mybatis-spring-boot-starter + 2.0.0 + + + + org.springframework.boot + spring-boot-starter-mail + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + org.springframework.boot + spring-boot-maven-plugin + + com.bfd.crawl.Application + ZIP + + + ${project.groupId} + ${project.artifactId} + + + + + + + repackage + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy + package + + copy-dependencies + + + jar + jar + runtime + ${project.build.directory}/libs + + + + + + + + \ No newline at end of file diff --git a/src/main/java/com/bfd/crawl/Application.java b/src/main/java/com/bfd/crawl/Application.java new file mode 100644 index 0000000..b10f6f8 --- /dev/null +++ b/src/main/java/com/bfd/crawl/Application.java @@ -0,0 +1,27 @@ +package com.bfd.crawl;//import com.bfd.crawl.process.ElasticsearchMaoJian; + +import com.bfd.crawl.process.ipManger; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * @author:zhaoying + * @className:Application + * @version:1.0 + * @description:主入口 + * @Date:2023-12-15 14:30:41 + */ +@SpringBootApplication +@EnableScheduling +@Slf4j +public class Application { + + public static void main(String[] args) { + SpringApplication.run(com.bfd.crawl.Application.class, args); + ipManger ipManger = new ipManger(); + Thread ipMangerThread = new Thread(ipManger); + ipMangerThread.start(); + } +} diff --git a/src/main/java/com/bfd/crawl/controller/GetDataController.java b/src/main/java/com/bfd/crawl/controller/GetDataController.java new file mode 100644 index 0000000..96de494 --- /dev/null +++ b/src/main/java/com/bfd/crawl/controller/GetDataController.java @@ -0,0 +1,162 @@ +package com.bfd.crawl.controller; + +import com.bfd.crawl.model.Ips; +import com.bfd.crawl.util.SendEmail; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @PROJECT_NAME: + * @DESCRIPTION:获取一个时效内的ip + * @AUTHOR: ying.zhao + * @DATE: 2023/4/3 18:17 + */ +@Slf4j +@RestController + +public class GetDataController { + @Autowired + private SendEmail sendEmail; + private static final Lock lock = new ReentrantLock(); + + @GetMapping(value = "/getIp") + public String testFrom() { +// 静态锁，所有请求共享 + String ip = ""; + String html = ""; +// while (ip.equals("")) { + Long randomKey = getRandomEligibleKey(); + if (randomKey != null) { + log.info("时效内的key: {}", randomKey); + ip = Ips.ipMap.get(randomKey); + log.info("时效内的ip: {}", ip); + } else { + if (lock.tryLock()) { + try { + // 执行你的业务逻辑 + if (Ips.ipMap.size() < 1) { + html = getIpList(); + if(html.contains("msg")){ + log.info("ip请求异常"); + return html; + } + } else { + log.info("已获取一次iP"); + } + } finally { + // 确保在执行完后释放锁 + lock.unlock(); + } + } else { + log.info("锁上了 等会儿吧。。。。。。。。。"); + } + } + return ip; + } + + /** + * @Param: + * @DESCRIPTION: 请求获取ip + * @Author: ying.zhao + * @date: 2024/9/12 + */ + public String getIpList() { + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + builder.readTimeout(200, TimeUnit.SECONDS); + builder.connectTimeout(200, TimeUnit.SECONDS); + OkHttpClient client = new OkHttpClient().newBuilder() + .build(); + MediaType mediaType = MediaType.parse("text/plain"); + RequestBody body = RequestBody.create(mediaType, ""); + Request request = new Request.Builder() + .url("http://api.tianqiip.com/getip?secret=x7it4ebj&num=10&type=txt&port=1&mr=1&sign=899ce225ae2cd14731d60fe48c338d0a") + .method("GET", null) + .build(); + Response response = null; + String html = ""; + try { + response = client.newCall(request).execute(); + html = response.body().string(); + } catch (IOException e) { + e.printStackTrace(); + } + if (!html.equals("")) { + log.info("请求得到页面：{}", html); + if (html.contains("msg")) { + try { + sendEmail.send("Alarm", "ip请求异常报警！！！！！！！！", html); + } catch (Exception e) { + e.printStackTrace(); + } + } else { + // 按换行符拆分字符串 + Ips.ipList = new ArrayList(Arrays.asList(html.split("\n"))); + for (String ip : Ips.ipList) { + log.info("ip：{}", ip); + Ips.ipMap.put(System.currentTimeMillis(), ip); + try { + Thread.sleep(1); // 延迟1毫秒 + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); // 处理中断异常 + } + } + } + + + } + return html; + + } + + /** + * @Param: + * @DESCRIPTION: 随机获取一个eligibleKeys中的key，确保每轮都被取到一次 + * @Author: ying.zhao + * @date: 2024/9/12 + */ + public static Long getRandomEligibleKey() { + if (Ips.eligibleKeys.isEmpty()) { + return null; // 如果列表为空，返回null + } + + // 如果所有符合条件的key都已经被取用，重新填充轮次 + if (Ips.keysInCurrentRound.size() == Ips.eligibleKeys.size()) { + Ips.keysInCurrentRound.clear(); // 清空当前轮次的标记 + } + + // 筛选出尚未被取用的key + List remainingKeys = new ArrayList<>(); + for (Long key : Ips.eligibleKeys) { + if (!Ips.keysInCurrentRound.contains(key)) { + remainingKeys.add(key); + } + } + + // 如果没有剩余的key可供选择 + if (remainingKeys.isEmpty()) { + return null; // 所有key都已被取用 + } + + // 随机选择一个尚未被取用的key + Random random = new Random(); + int randomIndex = random.nextInt(remainingKeys.size()); + Long randomKey = remainingKeys.get(randomIndex); + + // 标记已取用的key + Ips.keysInCurrentRound.add(randomKey); + + return randomKey; + } +} diff --git a/src/main/java/com/bfd/crawl/model/Ips.java b/src/main/java/com/bfd/crawl/model/Ips.java new file mode 100644 index 0000000..6a17105 --- /dev/null +++ b/src/main/java/com/bfd/crawl/model/Ips.java @@ -0,0 +1,22 @@ +package com.bfd.crawl.model; + +import lombok.Data; + +import java.util.*; +import java.util.logging.Handler; + +/** + * @PROJECT_NAME: + * @DESCRIPTION: + * @AUTHOR: ying.zhao + * @DATE: 2023/4/4 9:39 + */ +@Data +public class Ips { + public static List ipList = new ArrayList<>(); + public static Map ipMap = new HashMap(16); + // 临时列表存储符合条件的key + public static List eligibleKeys = new ArrayList<>(); + // 用于标记当前轮次中已取用的key + public static Set keysInCurrentRound = Collections.synchronizedSet(new HashSet<>()); +} diff --git a/src/main/java/com/bfd/crawl/process/ipManger.java b/src/main/java/com/bfd/crawl/process/ipManger.java new file mode 100644 index 0000000..ea773e3 --- /dev/null +++ b/src/main/java/com/bfd/crawl/process/ipManger.java @@ -0,0 +1,54 @@ +package com.bfd.crawl.process; + +import com.bfd.crawl.model.Ips; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * @PROJECT_NAME: ipPool_Manger + * @DESCRIPTION: + * @AUTHOR: ying.zhao + * @DATE: 2024/9/12 14:22 + */ +@Slf4j +public class ipManger implements Runnable { + @Override + public void run() { +// long tenMinutesInMillis = 3 * 60 * 1000; // 10分钟对应的毫秒数 + long tenMinutesInMillis =(long) (2.7 * 60 * 1000); // 2.7分钟对应的毫秒数 + while (true) { + log.info("正在监控ipMap中一共有：{}个", Ips.ipMap.size()); + log.info("正在监控可用ip有：{}个", Ips.eligibleKeys.size()); + long currentTime = System.currentTimeMillis(); // 当前时间戳 + if (Ips.ipMap.size() > 0) { + // 添加符合条件的key + for (Long key : Ips.ipMap.keySet()) { + if (currentTime - key <= tenMinutesInMillis && !Ips.eligibleKeys.contains(key)) { + Ips.eligibleKeys.add(key); // 只添加还未加入的key + } + } + + // 找到超过10分钟的key，先做入库操作 +// List keysToBeRemoved = Ips.eligibleKeys.stream() +// .filter(key -> currentTime - key > tenMinutesInMillis) +// .collect(Collectors.toList()); + +// if (!keysToBeRemoved.isEmpty()) { +//// saveToDatabase(keysToBeRemoved); // 保存这些key到数据库 +// } + + // 移除超过10分钟的key + Ips.eligibleKeys.removeIf(key -> currentTime - key > tenMinutesInMillis); + Ips.ipMap.entrySet().removeIf(entry -> currentTime - entry.getKey() > tenMinutesInMillis); + } + // 休眠一段时间，避免频繁循环 + try { + Thread.sleep(2 * 1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } +} diff --git a/src/main/java/com/bfd/crawl/util/SendEmail.java b/src/main/java/com/bfd/crawl/util/SendEmail.java new file mode 100644 index 0000000..a12f925 --- /dev/null +++ b/src/main/java/com/bfd/crawl/util/SendEmail.java @@ -0,0 +1,69 @@ +package com.bfd.crawl.util; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.Session; +import javax.mail.Transport; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; +import java.io.UnsupportedEncodingException; +import java.util.Date; +import java.util.Properties; + +/** + * @author:zhaoying + * @className:SendEmail + * @version:1.0 + * @description: + * @Date:2023-08-03 18:19:41 + */ + +@Slf4j +@Component +public class SendEmail { + @Value("${spring.email.host}") + private String host; + @Value("${spring.email.auth}") + private String auth; + @Value("${spring.email.user}") + private String user; + @Value("${spring.email.password}") + private String password; + @Value("${spring.email.receive}") + private String receive; + public void send(String sendUser, String title, String msage) throws MessagingException, UnsupportedEncodingException { + Properties props = new Properties(); + props.put("mail.smtp.host",host); + props.put("mail.smtp.auth", auth); + Session session = Session.getDefaultInstance(props, null); + Transport transport = session.getTransport(); + transport.connect(user, password); + + MimeMessage msg = new MimeMessage(session); + msg.setSentDate(new Date()); + //邮件发送人 + InternetAddress fromAddress = new InternetAddress(user, sendUser); + msg.setFrom(fromAddress); + //邮件接收人 + String[] receives = receive.split("#@#"); + for (int i=0;i + + + + + + + %d{HH:mm:ss.SSS} %-5level %logger{80} - %msg%n + + + + + true + + ${logging.level} + + + + ${logging.path}/data.log + + + ${logging.path}/data.log.%d{yyyy-MM-dd} + 7 + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %line %-5level %logger{50} - %msg%n + UTF-8 + + + + + + + +