package com.bfd.qanda.utils; import java.io.IOException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.LinkedList; import java.util.List; import java.util.Map; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.StatusLine; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.AuthCache; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.config.SocketConfig; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.LayeredConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.StringEntity; import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.LaxRedirectStrategy; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.bfd.qanda.entity.Constants; /** * 下载工具类 * @author jian.mao * @date 2023年9月19日 * @description */ public class DownLoadUtil { private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"; private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class); /** 代理服务器(产品官网 www.16yun.cn) **/ final static String PROXYHOST = "u270.40.tp.16yun.cn"; final static Integer PROXYPORT = 6448; /** 代理验证信息 **/ final static String PROXYUSER = "16HFBVJC"; final static String PROXYPASS = "897944"; private static PoolingHttpClientConnectionManager cm = null; private static HttpRequestRetryHandler httpRequestRetryHandler = null; private static HttpHost proxy = null; private static CredentialsProvider credsProvider = null; private static RequestConfig reqConfig = null; static { ConnectionSocketFactory plainsf = PlainConnectionSocketFactory .getSocketFactory(); LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory .getSocketFactory(); Registry registry = RegistryBuilder.create().register("http", plainsf) .register("https", sslsf).build(); cm = new PoolingHttpClientConnectionManager(registry); cm.setMaxTotal(20); cm.setDefaultMaxPerRoute(5); proxy = new HttpHost(PROXYHOST, PROXYPORT, "https"); credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(PROXYUSER, PROXYPASS)); reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000) .setConnectTimeout(5000).setSocketTimeout(5000) .setExpectContinueEnabled(false) .setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build(); } /** * 模拟客户端get请求 * * @param url * 模拟请求得url * @param headers * 头部信息,没有可以不传 * @return */ @SafeVarargs public static String proxyDoGet(String url, Map... headers) { // 设置超时时间 int timeout = 30; RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) .setConnectionRequestTimeout(timeout * 1000) .setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(timeout * 1000) .setTcpNoDelay(true).build(); AuthCache authCache = new BasicAuthCache(); authCache.put(proxy, new BasicScheme()); HttpClientContext localContext = HttpClientContext.create(); localContext.setAuthCache(authCache); HttpClientBuilder httpBuilder = HttpClientBuilder.create(); CloseableHttpClient httpClient = httpBuilder .setDefaultSocketConfig(socketConfig) .setDefaultRequestConfig(config) .setDefaultCredentialsProvider(credsProvider).build(); HttpGet httpGet = new HttpGet(url); httpGet.setConfig(reqConfig); if (headers != null && headers.length > 0) { Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpGet.setHeader(key, tempHeaders.get(key).toString()); } } else { httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); } CloseableHttpResponse response = null; String html = ""; int notFundCode = 404; int successCode = 200; try { response = httpClient.execute(httpGet, localContext); // 从响应模型中获取响应实体 HttpEntity responseEntity = response.getEntity(); StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); if (statusLine.getStatusCode() == successCode) { if (responseEntity != null) { html = EntityUtils.toString(responseEntity, "utf-8"); System.out.println("响应内容长度为:" + responseEntity.getContentLength()); // 下载结果为空不正常 if (html.equals(Constants.EMPTY)) { html = "Download failed error is:reslut is null"; } } } else if (statusLine.getStatusCode() == notFundCode) { html = "

页面404,正常结束请求即可

"; } else { throw new Exception("请求错误,code码为:" + statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:reslut is null"; }finally{ try { response.close(); httpClient.close(); } catch (Exception e) { e.printStackTrace(); } } return html; } public static String httpsslProxyGet(String url, Map... headers) throws Exception { //采用绕过验证的方式处理https请求 SSLContext sslcontext = createIgnoreVerifySSL(); // 设置协议http和https对应的处理socket链接工厂的对象 Registry socketFactoryRegistry = RegistryBuilder.create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext)) .build(); PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); connManager.setMaxTotal(50); connManager.setDefaultMaxPerRoute(10); HttpClients.custom().setConnectionManager(connManager); // 设置超时时间 int timeout = 30; RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) .setConnectionRequestTimeout(timeout * 1000) .setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(timeout * 1000) .setTcpNoDelay(true).build(); AuthCache authCache = new BasicAuthCache(); authCache.put(proxy, new BasicScheme()); HttpClientContext localContext = HttpClientContext.create(); localContext.setAuthCache(authCache); HttpClientBuilder httpBuilder = HttpClientBuilder.create(); CloseableHttpClient httpClient = httpBuilder .setConnectionManager(connManager) .setDefaultSocketConfig(socketConfig) .setDefaultRequestConfig(config) .setDefaultCredentialsProvider(credsProvider).build(); HttpGet httpGet = new HttpGet(url); httpGet.setConfig(reqConfig); if (headers != null && headers.length > 0) { Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpGet.setHeader(key, tempHeaders.get(key).toString()); } } else { httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); } CloseableHttpResponse response = null; String html = ""; int notFundCode = 404; int successCode = 200; try { response = httpClient.execute(httpGet, localContext); // 从响应模型中获取响应实体 HttpEntity responseEntity = response.getEntity(); StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); if (statusLine.getStatusCode() == successCode) { if (responseEntity != null) { html = EntityUtils.toString(responseEntity, "utf-8"); System.out.println("响应内容长度为:" + responseEntity.getContentLength()); // 下载结果为空不正常 if (html.equals(Constants.EMPTY)) { html = "Download failed error is:reslut is null"; } } } else if (statusLine.getStatusCode() == notFundCode) { html = "

页面404,正常结束请求即可

"; } else { throw new Exception("请求错误,code码为:" + statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:reslut is null"; }finally{ try { response.close(); httpClient.close(); } catch (Exception e) { e.printStackTrace(); } } return html; } /** * json参数方式POST提交 * @param url * @param params * @return */ public static String doPost(String url, String params, Map... headers){ String strResult = ""; //设置超时时间 int timeout = 60; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(timeout * 1000) .setTcpNoDelay(true).build(); // AuthCache authCache = new BasicAuthCache(); // authCache.put(proxy, new BasicScheme()); // HttpClientContext localContext = HttpClientContext.create(); // localContext.setAuthCache(authCache); // 1. 获取默认的client实例 HttpClientBuilder httpBuilder = HttpClientBuilder.create(); httpBuilder.setUserAgent(ua); HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); // HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) // .setDefaultCredentialsProvider(credsProvider).build(); // 2. 创建httppost实例 HttpPost httpPost = new HttpPost(url); // httpPost.setConfig(reqConfig); if (headers != null && headers.length > 0) { Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpPost.setHeader(key, tempHeaders.get(key).toString()); } } else { httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); } HttpResponse resp = null; try { httpPost.setEntity(new StringEntity(params,"utf-8")); resp = client.execute(httpPost); // resp = client.execute(httpPost,localContext); StatusLine statusLine = resp.getStatusLine(); System.out.println("响应状态为:" + resp.getStatusLine()); int notFundCode = 404; int successCode = 200; if(statusLine.getStatusCode() == successCode){ // 7. 获取响应entity HttpEntity respEntity = resp.getEntity(); strResult = EntityUtils.toString(respEntity, "UTF-8"); if(strResult.equals(Constants.EMPTY)){ strResult = "Download failed error is:reslut is null"; } }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } return strResult; } public static String httpPost(String url,String params) { String html=""; html = doPost(url,params); int i = 1; while(true){ if(html.contains("Download failed error is:")){ log.error("DownLoadUtil------------->download is failure,url is:"+url); DateUtil.sleep(5000); i++; }else{ break; } if(i > 5){ break; } html = doPost(url,params); } return html; } /** * 绕过验证 * * @return * @throws NoSuchAlgorithmException * @throws KeyManagementException */ public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException { SSLContext sc = SSLContext.getInstance("SSLv3"); // 实现一个X509TrustManager接口,用于绕过验证,不用修改里面的方法 X509TrustManager trustManager = new X509TrustManager() { @Override public void checkClientTrusted( java.security.cert.X509Certificate[] paramArrayOfX509Certificate, String paramString) throws CertificateException { } @Override public void checkServerTrusted( java.security.cert.X509Certificate[] paramArrayOfX509Certificate, String paramString) throws CertificateException { } @Override public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; } }; sc.init(null, new TrustManager[] { trustManager }, null); return sc; } /** * 模拟请求 * * @param url 资源地址 * @param map 参数列表 * @param encoding 编码 * @return * @throws NoSuchAlgorithmException * @throws KeyManagementException * @throws IOException * @throws ClientProtocolException */ public static String httpsslGet(String url,Map ... headers) { String html=""; CloseableHttpClient client = null; HttpEntity responseEntity = null; CloseableHttpResponse response = null; try { log.debug("DownLoadUtil------------->设置下载相关信息, start...."); //采用绕过验证的方式处理https请求 SSLContext sslcontext = createIgnoreVerifySSL(); // 设置协议http和https对应的处理socket链接工厂的对象 Registry socketFactoryRegistry = RegistryBuilder.create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext)) .build(); PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); connManager.setMaxTotal(50); connManager.setDefaultMaxPerRoute(10); HttpClients.custom().setConnectionManager(connManager); //设置超时时间 int timeout = 30; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(10000) .setTcpNoDelay(true).build(); // 设置重定向策略 LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy(); //创建自定义的httpclient对象 client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build(); // CloseableHttpClient client = HttpClients.createDefault(); HttpGet httpGet = new HttpGet(url); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpGet.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); } log.debug("DownLoadUtil------------->设置下载相关信息, end...."); try { int notFundCode = 404; int successCode = 200; log.debug("DownLoadUtil------------->下载执行,start...."); httpGet.setConfig(config); response = client.execute(httpGet); log.debug("DownLoadUtil------------->下载执行,end...."); // 从响应模型中获取响应实体 StatusLine statusLine = response.getStatusLine(); log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+",read is start ...."); System.out.println("响应状态为:" + response.getStatusLine()); responseEntity = response.getEntity(); log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+",read is end ...."); if(statusLine.getStatusCode() == successCode){ if (responseEntity != null) { html=EntityUtils.toString(responseEntity,"utf-8"); System.out.println("响应内容长度为:" + responseEntity.getContentLength()); } }else if(statusLine.getStatusCode() == notFundCode){ html = "

页面404,正常结束请求即可

"; }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); }finally{ try { responseEntity.getContent().close(); response.close(); client.close(); } catch (Exception e) { e.printStackTrace(); } } return html; } public static String httpSSLGet(String url,Map ... headers) { String html=""; html = httpsslGet(url,headers); int i = 1; while(true){ if(html.contains("Download failed error is:")){ log.error("DownLoadUtil------------->download is failure,url is:"+url); DateUtil.sleep(30000); i++; }else{ break; } if(i > 5){ break; } html = httpsslGet(url,headers); } return html; } public static String doPostFrom(String url,Map param,Map ... headers){ //设置超时时间 int timeout = 15; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(10000) .setTcpNoDelay(true).build(); // AuthCache authCache = new BasicAuthCache(); // authCache.put(proxy, new BasicScheme()); // HttpClientContext localContext = HttpClientContext.create(); // localContext.setAuthCache(authCache); HttpClientBuilder httpBuilder = HttpClientBuilder.create(); httpBuilder.setUserAgent(ua); // HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) // .setDefaultCredentialsProvider(credsProvider).build(); HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); HttpPost httpPost = new HttpPost(url); // httpPost.setConfig(reqConfig); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpPost.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"); httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"); // httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html"); } // 创建请求参数 List list = new LinkedList<>(); for (String key : param.keySet()) { BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); list.add(param1); } // 使用URL实体转换工具 String html=""; try { UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); httpPost.setEntity(entityParam); HttpResponse response = httpClient.execute(httpPost); // HttpResponse response = httpClient.execute(httpPost,localContext); // 从响应模型中获取响应实体 HttpEntity responseEntity = response.getEntity(); StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); int notFundCode = 404; int successCode = 200; if(statusLine.getStatusCode() == successCode){ if (responseEntity != null) { html=EntityUtils.toString(responseEntity,"utf-8"); } }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } return html; } public static String httpPostForm(String url,Map params,Map ... headers) { String html=""; html = doPostFrom(url,params); int i = 1; while(true){ if(html.contains("Download failed error is:")){ log.error("DownLoadUtil------------->download is failure,url is:"+url); DateUtil.sleep(5000); i++; }else{ break; } if(i > 5){ break; } html = doPostFrom(url,params,headers); } return html; } public static String dosslPost(String url,String params,Map ... headers) { String html=""; CloseableHttpClient client = null; HttpEntity responseEntity = null; CloseableHttpResponse response = null; try { //采用绕过验证的方式处理https请求 SSLContext sslcontext = createIgnoreVerifySSL(); // 设置协议http和https对应的处理socket链接工厂的对象 Registry socketFactoryRegistry = RegistryBuilder.create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext)) .build(); PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); HttpClients.custom().setConnectionManager(connManager); //设置超时时间 int timeout = 5; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(10000) .setTcpNoDelay(true).build(); //创建自定义的httpclient对象 client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); // CloseableHttpClient client = HttpClients.createDefault(); // 2. 创建httppost实例 HttpPost httpPost = new HttpPost(url); // httpPost.setConfig(reqConfig); httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpPost.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); } try { httpPost.setEntity(new StringEntity(params,"utf-8")); response = client.execute(httpPost); int notFundCode = 404; int successCode = 200; // 从响应模型中获取响应实体 StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); responseEntity = response.getEntity(); if(statusLine.getStatusCode() == successCode){ if (responseEntity != null) { html=EntityUtils.toString(responseEntity,"utf-8"); System.out.println("响应内容长度为:" + responseEntity.getContentLength()); } }else if(statusLine.getStatusCode() == notFundCode){ html = "

页面404,正常结束请求即可

"; }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); }finally{ try { responseEntity.getContent().close(); response.close(); client.close(); } catch (UnsupportedOperationException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } return html; } public static String dosslPostForm(String url,Map param,Map ... headers) { String html=""; try { //采用绕过验证的方式处理https请求 SSLContext sslcontext = createIgnoreVerifySSL(); // 设置协议http和https对应的处理socket链接工厂的对象 Registry socketFactoryRegistry = RegistryBuilder.create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext)) .build(); PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); HttpClients.custom().setConnectionManager(connManager); //设置超时时间 int timeout = 5; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(10000) .setTcpNoDelay(true).build(); //创建自定义的httpclient对象 CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); // CloseableHttpClient client = HttpClients.createDefault(); // 2. 创建httppost实例 HttpPost httpPost = new HttpPost(url); // httpPost.setConfig(reqConfig); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpPost.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); } // 创建请求参数 List list = new LinkedList<>(); for (String key : param.keySet()) { BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); list.add(param1); } // 使用URL实体转换工具 try { UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); httpPost.setEntity(entityParam); HttpResponse response = client.execute(httpPost); // HttpResponse response = httpClient.execute(httpPost,localContext); // 从响应模型中获取响应实体 int notFundCode = 404; int successCode = 200; HttpEntity responseEntity = response.getEntity(); StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); if(statusLine.getStatusCode() == successCode){ if (responseEntity != null) { html=EntityUtils.toString(responseEntity,"utf-8"); } }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } return html; } public static String httpSSLPostForm(String url,Map params,Map ...headers) { String html=""; try { html = dosslPostForm(url,params,headers); } catch (Exception e) { e.printStackTrace(); // TODO: handle exception html = "Download failed error is:Exception!"; } int i = 1; while(true){ if(html.contains("Download failed error is:")){ log.error("DownLoadUtil------------->download is failure,url is:"+url); DateUtil.sleep(30000); i++; }else{ break; } if(i > 5){ break; } try { html = dosslPostForm(url,params,headers); } catch (Exception e) { e.printStackTrace(); // TODO: handle exception html = "Download failed error is:Exception!"; } } return html; } public static String httpSSLPost(String url,String params,Map ...headers) { String html=""; try { html = dosslPost(url,params,headers); } catch (Throwable e) { e.printStackTrace(); // TODO: handle exception html = "Download failed error is:Exception!"; } int i = 1; while(true){ if(html.contains("Download failed error is:")){ log.error("DownLoadUtil------------->download is failure,url is:"+url); DateUtil.sleep(30000); i++; }else{ break; } if(i > 5){ break; } try { html = dosslPost(url,params,headers); } catch (Throwable e) { e.printStackTrace(); // TODO: handle exception html = "Download failed error is:Exception!"; } } return html; } /** * 模拟客户端get请求 * @param url 模拟请求得url * @param headers 头部信息,没有可以不传 * @return */ public static String doGet(String url,Map ... headers){ //设置超时时间 int timeout = 15; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(10000) .setTcpNoDelay(true).build(); HttpClientBuilder httpBuilder = HttpClientBuilder.create(); httpBuilder.setUserAgent(ua); HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); HttpGet httpGet = new HttpGet(url); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpGet.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); } String html=""; try { int notFundCode = 404; int successCode = 200; HttpResponse response = httpClient.execute(httpGet); // 从响应模型中获取响应实体 HttpEntity responseEntity = response.getEntity(); StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); if(statusLine.getStatusCode() == successCode){ if (responseEntity != null) { html=EntityUtils.toString(responseEntity,"utf-8"); if(html.equals("")){ html = "Download failed error is:reslut is null"; } } }else if(statusLine.getStatusCode() == notFundCode){ html = "

页面404,正常结束请求即可

"; }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } return html; } public static void main(String[] args) throws Exception { } }