package com.bfd.qanda.utils; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.LinkedList; import java.util.List; import java.util.Map; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.StatusLine; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.AuthCache; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.config.SocketConfig; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.LayeredConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.LaxRedirectStrategy; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.bfd.qanda.entity.Constants; /** * 下载工具类 * @author jian.mao * @date 2023年9月19日 * @description */ public class DownLoadUtil { private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"; private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class); /** 代理服务器(产品官网 www.16yun.cn) **/ final static String PROXYHOST = "u270.40.tp.16yun.cn"; final static Integer PROXYPORT = 6448; /** 代理验证信息 **/ final static String PROXYUSER = "16HFBVJC"; final static String PROXYPASS = "897944"; private static PoolingHttpClientConnectionManager cm = null; private static HttpRequestRetryHandler httpRequestRetryHandler = null; private static HttpHost proxy = null; private static CredentialsProvider credsProvider = null; private static RequestConfig reqConfig = null; static { ConnectionSocketFactory plainsf = PlainConnectionSocketFactory .getSocketFactory(); LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory .getSocketFactory(); Registry registry = RegistryBuilder.create().register("http", plainsf) .register("https", sslsf).build(); cm = new PoolingHttpClientConnectionManager(registry); cm.setMaxTotal(20); cm.setDefaultMaxPerRoute(5); proxy = new HttpHost(PROXYHOST, PROXYPORT, "https"); credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(PROXYUSER, PROXYPASS)); reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000) .setConnectTimeout(5000).setSocketTimeout(5000) .setExpectContinueEnabled(false) .setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build(); } /** * 模拟客户端get请求 * * @param url * 模拟请求得url * @param headers * 头部信息,没有可以不传 * @return */ @SafeVarargs public static String proxyDoGet(String url, Map... headers) { // 设置超时时间 int timeout = 30; RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) .setConnectionRequestTimeout(timeout * 1000) .setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(timeout * 1000) .setTcpNoDelay(true).build(); AuthCache authCache = new BasicAuthCache(); authCache.put(proxy, new BasicScheme()); HttpClientContext localContext = HttpClientContext.create(); localContext.setAuthCache(authCache); HttpClientBuilder httpBuilder = HttpClientBuilder.create(); CloseableHttpClient httpClient = httpBuilder .setDefaultSocketConfig(socketConfig) .setDefaultRequestConfig(config) .setDefaultCredentialsProvider(credsProvider).build(); HttpGet httpGet = new HttpGet(url); httpGet.setConfig(reqConfig); if (headers != null && headers.length > 0) { Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpGet.setHeader(key, tempHeaders.get(key).toString()); } } else { httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); } CloseableHttpResponse response = null; String html = ""; int notFundCode = 404; int successCode = 200; try { response = httpClient.execute(httpGet, localContext); // 从响应模型中获取响应实体 HttpEntity responseEntity = response.getEntity(); StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); if (statusLine.getStatusCode() == successCode) { if (responseEntity != null) { html = EntityUtils.toString(responseEntity, "utf-8"); System.out.println("响应内容长度为:" + responseEntity.getContentLength()); // 下载结果为空不正常 if (html.equals(Constants.EMPTY)) { html = "Download failed error is:reslut is null"; } } } else if (statusLine.getStatusCode() == notFundCode) { html = "

页面404,正常结束请求即可

"; } else { throw new Exception("请求错误,code码为:" + statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:reslut is null"; }finally{ try { response.close(); httpClient.close(); } catch (Exception e) { e.printStackTrace(); } } return html; } public static String httpsslProxyGet(String url, Map... headers) throws Exception { //采用绕过验证的方式处理https请求 SSLContext sslcontext = createIgnoreVerifySSL(); // 设置协议http和https对应的处理socket链接工厂的对象 Registry socketFactoryRegistry = RegistryBuilder.create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext)) .build(); PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); connManager.setMaxTotal(50); connManager.setDefaultMaxPerRoute(10); HttpClients.custom().setConnectionManager(connManager); // 设置超时时间 int timeout = 30; RequestConfig config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) .setConnectionRequestTimeout(timeout * 1000) .setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(timeout * 1000) .setTcpNoDelay(true).build(); AuthCache authCache = new BasicAuthCache(); authCache.put(proxy, new BasicScheme()); HttpClientContext localContext = HttpClientContext.create(); localContext.setAuthCache(authCache); HttpClientBuilder httpBuilder = HttpClientBuilder.create(); CloseableHttpClient httpClient = httpBuilder .setConnectionManager(connManager) .setDefaultSocketConfig(socketConfig) .setDefaultRequestConfig(config) .setDefaultCredentialsProvider(credsProvider).build(); HttpGet httpGet = new HttpGet(url); httpGet.setConfig(reqConfig); if (headers != null && headers.length > 0) { Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpGet.setHeader(key, tempHeaders.get(key).toString()); } } else { httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); } CloseableHttpResponse response = null; String html = ""; int notFundCode = 404; int successCode = 200; try { response = httpClient.execute(httpGet, localContext); // 从响应模型中获取响应实体 HttpEntity responseEntity = response.getEntity(); StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); if (statusLine.getStatusCode() == successCode) { if (responseEntity != null) { html = EntityUtils.toString(responseEntity, "utf-8"); System.out.println("响应内容长度为:" + responseEntity.getContentLength()); // 下载结果为空不正常 if (html.equals(Constants.EMPTY)) { html = "Download failed error is:reslut is null"; } } } else if (statusLine.getStatusCode() == notFundCode) { html = "

页面404,正常结束请求即可

"; } else { throw new Exception("请求错误,code码为:" + statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:reslut is null"; }finally{ try { response.close(); httpClient.close(); } catch (Exception e) { e.printStackTrace(); } } return html; } /** * json参数方式POST提交 * @param url * @param params * @return */ public static String doPost(String url, String params, Map... headers){ String strResult = ""; //设置超时时间 int timeout = 180; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(timeout * 1000) .setTcpNoDelay(true).build(); // AuthCache authCache = new BasicAuthCache(); // authCache.put(proxy, new BasicScheme()); // HttpClientContext localContext = HttpClientContext.create(); // localContext.setAuthCache(authCache); // 1. 获取默认的client实例 HttpClientBuilder httpBuilder = HttpClientBuilder.create(); httpBuilder.setUserAgent(ua); HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); // HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) // .setDefaultCredentialsProvider(credsProvider).build(); // 2. 创建httppost实例 HttpPost httpPost = new HttpPost(url); // httpPost.setConfig(reqConfig); if (headers != null && headers.length > 0) { Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpPost.setHeader(key, tempHeaders.get(key).toString()); } } else { httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); } HttpResponse resp = null; try { httpPost.setEntity(new StringEntity(params,"utf-8")); resp = client.execute(httpPost); // resp = client.execute(httpPost,localContext); StatusLine statusLine = resp.getStatusLine(); System.out.println("响应状态为:" + resp.getStatusLine()); int notFundCode = 404; int successCode = 200; if(statusLine.getStatusCode() == successCode){ // 7. 获取响应entity HttpEntity respEntity = resp.getEntity(); strResult = EntityUtils.toString(respEntity, "UTF-8"); if(strResult.equals(Constants.EMPTY)){ strResult = "Download failed error is:reslut is null"; } }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } return strResult; } public static String httpPost(String url,String params) { String html=""; html = doPost(url,params); int i = 1; while(true){ if(html.contains("Download failed error is:")){ log.error("DownLoadUtil------------->download is failure,url is:"+url); DateUtil.sleep(5000); i++; }else{ break; } if(i > 5){ break; } html = doPost(url,params); } return html; } /** * 绕过验证 * * @return * @throws NoSuchAlgorithmException * @throws KeyManagementException */ public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException { SSLContext sc = SSLContext.getInstance("SSLv3"); // 实现一个X509TrustManager接口,用于绕过验证,不用修改里面的方法 X509TrustManager trustManager = new X509TrustManager() { @Override public void checkClientTrusted( java.security.cert.X509Certificate[] paramArrayOfX509Certificate, String paramString) throws CertificateException { } @Override public void checkServerTrusted( java.security.cert.X509Certificate[] paramArrayOfX509Certificate, String paramString) throws CertificateException { } @Override public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; } }; sc.init(null, new TrustManager[] { trustManager }, null); return sc; } /** * 模拟请求 * * @param url 资源地址 * @param map 参数列表 * @param encoding 编码 * @return * @throws NoSuchAlgorithmException * @throws KeyManagementException * @throws IOException * @throws ClientProtocolException */ public static String httpsslGet(String url,Map ... headers) { String html=""; CloseableHttpClient client = null; HttpEntity responseEntity = null; CloseableHttpResponse response = null; try { log.debug("DownLoadUtil------------->设置下载相关信息, start...."); //采用绕过验证的方式处理https请求 SSLContext sslcontext = createIgnoreVerifySSL(); // 设置协议http和https对应的处理socket链接工厂的对象 Registry socketFactoryRegistry = RegistryBuilder.create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext)) .build(); PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); connManager.setMaxTotal(50); connManager.setDefaultMaxPerRoute(10); HttpClients.custom().setConnectionManager(connManager); //设置超时时间 int timeout = 30; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(10000) .setTcpNoDelay(true).build(); // 设置重定向策略 LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy(); //创建自定义的httpclient对象 client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build(); // CloseableHttpClient client = HttpClients.createDefault(); HttpGet httpGet = new HttpGet(url); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpGet.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); } log.debug("DownLoadUtil------------->设置下载相关信息, end...."); try { int notFundCode = 404; int successCode = 200; log.debug("DownLoadUtil------------->下载执行,start...."); httpGet.setConfig(config); response = client.execute(httpGet); log.debug("DownLoadUtil------------->下载执行,end...."); // 从响应模型中获取响应实体 StatusLine statusLine = response.getStatusLine(); log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+",read is start ...."); System.out.println("响应状态为:" + response.getStatusLine()); responseEntity = response.getEntity(); log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+",read is end ...."); if(statusLine.getStatusCode() == successCode){ if (responseEntity != null) { html=EntityUtils.toString(responseEntity,"utf-8"); System.out.println("响应内容长度为:" + responseEntity.getContentLength()); } }else if(statusLine.getStatusCode() == notFundCode){ html = "

页面404,正常结束请求即可

"; }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); }finally{ try { responseEntity.getContent().close(); response.close(); client.close(); } catch (Exception e) { e.printStackTrace(); } } return html; } public static String httpSSLGet(String url,Map ... headers) { String html=""; html = httpsslGet(url,headers); int i = 1; while(true){ if(html.contains("Download failed error is:")){ log.error("DownLoadUtil------------->download is failure,url is:"+url); DateUtil.sleep(30000); i++; }else{ break; } if(i > 5){ break; } html = httpsslGet(url,headers); } return html; } public static String doPostFrom(String url,Map param,Map ... headers){ //设置超时时间 int timeout = 15; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(10000) .setTcpNoDelay(true).build(); // AuthCache authCache = new BasicAuthCache(); // authCache.put(proxy, new BasicScheme()); // HttpClientContext localContext = HttpClientContext.create(); // localContext.setAuthCache(authCache); HttpClientBuilder httpBuilder = HttpClientBuilder.create(); httpBuilder.setUserAgent(ua); // HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm) // .setDefaultCredentialsProvider(credsProvider).build(); HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); HttpPost httpPost = new HttpPost(url); // httpPost.setConfig(reqConfig); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpPost.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"); httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"); // httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html"); } // 创建请求参数 List list = new LinkedList<>(); for (String key : param.keySet()) { BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); list.add(param1); } // 使用URL实体转换工具 String html=""; try { UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); httpPost.setEntity(entityParam); HttpResponse response = httpClient.execute(httpPost); // HttpResponse response = httpClient.execute(httpPost,localContext); // 从响应模型中获取响应实体 HttpEntity responseEntity = response.getEntity(); StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); int notFundCode = 404; int successCode = 200; if(statusLine.getStatusCode() == successCode){ if (responseEntity != null) { html=EntityUtils.toString(responseEntity,"utf-8"); } }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } return html; } public static String httpPostForm(String url,Map params,Map ... headers) { String html=""; html = doPostFrom(url,params); int i = 1; while(true){ if(html.contains("Download failed error is:")){ log.error("DownLoadUtil------------->download is failure,url is:"+url); DateUtil.sleep(5000); i++; }else{ break; } if(i > 5){ break; } html = doPostFrom(url,params,headers); } return html; } public static String dosslPost(String url,String params,Map ... headers) { String html=""; CloseableHttpClient client = null; HttpEntity responseEntity = null; CloseableHttpResponse response = null; try { //采用绕过验证的方式处理https请求 SSLContext sslcontext = createIgnoreVerifySSL(); // 设置协议http和https对应的处理socket链接工厂的对象 Registry socketFactoryRegistry = RegistryBuilder.create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext)) .build(); PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); HttpClients.custom().setConnectionManager(connManager); //设置超时时间 int timeout = 5; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(10000) .setTcpNoDelay(true).build(); //创建自定义的httpclient对象 client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); // CloseableHttpClient client = HttpClients.createDefault(); // 2. 创建httppost实例 HttpPost httpPost = new HttpPost(url); // httpPost.setConfig(reqConfig); httpPost.addHeader("Content-Type", "application/json;charset=utf-8"); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpPost.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); } try { httpPost.setEntity(new StringEntity(params,"utf-8")); response = client.execute(httpPost); int notFundCode = 404; int successCode = 200; // 从响应模型中获取响应实体 StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); responseEntity = response.getEntity(); if(statusLine.getStatusCode() == successCode){ if (responseEntity != null) { html=EntityUtils.toString(responseEntity,"utf-8"); System.out.println("响应内容长度为:" + responseEntity.getContentLength()); } }else if(statusLine.getStatusCode() == notFundCode){ html = "

页面404,正常结束请求即可

"; }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); }finally{ try { responseEntity.getContent().close(); response.close(); client.close(); } catch (UnsupportedOperationException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } return html; } public static String dosslPostForm(String url,Map param,Map ... headers) { String html=""; try { //采用绕过验证的方式处理https请求 SSLContext sslcontext = createIgnoreVerifySSL(); // 设置协议http和https对应的处理socket链接工厂的对象 Registry socketFactoryRegistry = RegistryBuilder.create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext)) .build(); PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); HttpClients.custom().setConnectionManager(connManager); //设置超时时间 int timeout = 5; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(10000) .setTcpNoDelay(true).build(); //创建自定义的httpclient对象 CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build(); // CloseableHttpClient client = HttpClients.createDefault(); // 2. 创建httppost实例 HttpPost httpPost = new HttpPost(url); // httpPost.setConfig(reqConfig); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpPost.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); httpPost.addHeader("content-type", "application/x-www-form-urlencoded"); httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); } // 创建请求参数 List list = new LinkedList<>(); for (String key : param.keySet()) { BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString()); list.add(param1); } // 使用URL实体转换工具 try { UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8"); httpPost.setEntity(entityParam); HttpResponse response = client.execute(httpPost); // HttpResponse response = httpClient.execute(httpPost,localContext); // 从响应模型中获取响应实体 int notFundCode = 404; int successCode = 200; HttpEntity responseEntity = response.getEntity(); StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); if(statusLine.getStatusCode() == successCode){ if (responseEntity != null) { html=EntityUtils.toString(responseEntity,"utf-8"); } }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } return html; } public static String httpSSLPostForm(String url,Map params,Map ...headers) { String html=""; try { html = dosslPostForm(url,params,headers); } catch (Exception e) { e.printStackTrace(); // TODO: handle exception html = "Download failed error is:Exception!"; } int i = 1; while(true){ if(html.contains("Download failed error is:")){ log.error("DownLoadUtil------------->download is failure,url is:"+url); DateUtil.sleep(30000); i++; }else{ break; } if(i > 5){ break; } try { html = dosslPostForm(url,params,headers); } catch (Exception e) { e.printStackTrace(); // TODO: handle exception html = "Download failed error is:Exception!"; } } return html; } public static String httpSSLPost(String url,String params,Map ...headers) { String html=""; try { html = dosslPost(url,params,headers); } catch (Throwable e) { e.printStackTrace(); // TODO: handle exception html = "Download failed error is:Exception!"; } int i = 1; while(true){ if(html.contains("Download failed error is:")){ log.error("DownLoadUtil------------->download is failure,url is:"+url); DateUtil.sleep(30000); i++; }else{ break; } if(i > 5){ break; } try { html = dosslPost(url,params,headers); } catch (Throwable e) { e.printStackTrace(); // TODO: handle exception html = "Download failed error is:Exception!"; } } return html; } /** * 模拟客户端get请求 * @param url 模拟请求得url * @param headers 头部信息,没有可以不传 * @return */ public static String doGet(String url,Map ... headers){ //设置超时时间 int timeout = 15; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(10000) .setTcpNoDelay(true).build(); HttpClientBuilder httpBuilder = HttpClientBuilder.create(); httpBuilder.setUserAgent(ua); HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); HttpGet httpGet = new HttpGet(url); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpGet.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); } String html=""; try { int notFundCode = 404; int successCode = 200; HttpResponse response = httpClient.execute(httpGet); // 从响应模型中获取响应实体 HttpEntity responseEntity = response.getEntity(); StatusLine statusLine = response.getStatusLine(); System.out.println("响应状态为:" + response.getStatusLine()); if(statusLine.getStatusCode() == successCode){ if (responseEntity != null) { html=EntityUtils.toString(responseEntity,"utf-8"); if(html.equals("")){ html = "Download failed error is:reslut is null"; } } }else if(statusLine.getStatusCode() == notFundCode){ html = "

页面404,正常结束请求即可

"; }else{ throw new Exception("请求错误,code码为:"+statusLine.getStatusCode()); } } catch (Exception e) { e.printStackTrace(); html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e); } return html; } /** * 文件下载 * @param fileURL 文件连接 * @param saveFilePath 文件存储地址 * @throws IOException */ public static void downloadFile(String fileURL, String saveFilePath,Map ... headers) { //设置超时时间 int timeout = 60; RequestConfig config = RequestConfig.custom(). setConnectTimeout(timeout * 1000). setConnectionRequestTimeout(timeout * 1000). setSocketTimeout(timeout * 1000).build(); SocketConfig socketConfig = SocketConfig.custom() .setSoKeepAlive(false) .setSoLinger(1) .setSoReuseAddress(true) .setSoTimeout(timeout * 1000) .setTcpNoDelay(true).build(); HttpClientBuilder httpBuilder = HttpClientBuilder.create(); httpBuilder.setUserAgent(ua); HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build(); HttpGet httpGet = new HttpGet(fileURL); if(headers != null && headers.length > 0){ Map tempHeaders = headers[0]; for (String key : tempHeaders.keySet()) { httpGet.setHeader(key,tempHeaders.get(key).toString()); } }else{ httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8"); } try { int successCode = 200; HttpResponse response = httpClient.execute(httpGet); // 从响应模型中获取响应实体 StatusLine statusLine = response.getStatusLine(); int statusCode = statusLine.getStatusCode(); if(statusCode == successCode){ // 获取响应体 HttpEntity entity = response.getEntity(); if (entity != null) { // 创建输入流来读取文件内容 try (InputStream inputStream = entity.getContent(); FileOutputStream fileOutputStream = new FileOutputStream(new File(saveFilePath))) { byte[] buffer = new byte[4096]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { fileOutputStream.write(buffer, 0, bytesRead); } } log.info("文件下载成功,保存路径为: {}" , saveFilePath); } } else { log.error("文件下载失败,HTTP 响应状态码为: {}" , statusCode); } EntityUtils.consume(response.getEntity()); } catch (Exception e) { log.error("下载文件异常:{}",e); } } /** * 删除文件 * @param filePath */ public static void delFile(String filePath) { try { // 创建 File 对象 File file = new File(filePath); // 检查文件是否存在 if (file.exists()) { // 尝试删除文件 if (file.delete()) { log.info("文件删除成功: " + filePath); } else { log.error("无法删除文件: " + filePath); } } else { log.warn("文件不存在: " + filePath); } } catch (Exception e) { // TODO: handle exception log.error("删除文件异常:{}",filePath); } } /** * @param uploadUrl * @param filePath */ public static String uploadFile(String uploadUrl, String filePath) { // 创建HttpClient实例 CloseableHttpClient httpClient = HttpClients.createDefault(); String gofastUrl = ""; try { // 创建HttpPost请求 HttpPost httpPost = new HttpPost(uploadUrl); // 要上传的文件 File file = new File(filePath); if (!file.exists()) { log.error("文件不存在:{}" , filePath); return gofastUrl; } // 构建文件上传请求体 HttpEntity entity = MultipartEntityBuilder.create() .addBinaryBody("file", file, ContentType.DEFAULT_BINARY, file.getName()) .build(); // 设置请求体 httpPost.setEntity(entity); // 执行请求并获取响应 CloseableHttpResponse response = httpClient.execute(httpPost); try { int statusCode = response.getStatusLine().getStatusCode(); int successCode = 200; if (statusCode == successCode) { // 解析响应内容 gofastUrl = EntityUtils.toString(response.getEntity()); log.info("文件上传成功,响应:{}" , gofastUrl); } else { log.error("文件上传失败,HTTP 响应码:{}" , statusCode); } }catch (Exception e) { // TODO: handle exception } finally { try { response.close(); } catch (IOException e) { e.printStackTrace(); } } }catch(Exception e){ e.printStackTrace(); } finally { try { httpClient.close(); } catch (IOException e) { e.printStackTrace(); } } return gofastUrl; } public static void main(String[] args) throws Exception { // String url = "https://public-cdn-video-data-algeng.oss-cn-wulanchabu.aliyuncs.com/inference_output%2Fvideo%2F2024-09-14%2Fa13bd277-691e-4a3e-9096-ec5e9a449aa3%2Foutput.mp4?Expires=1726313297&OSSAccessKeyId=LTAI5tGLnRTkBjLuYPjNcKQ8&Signature=TmZDbYyisMoNi0smdEUCkzooTZI%3D"; String path = "D:\\123.mp4"; // downloadFile(url,path); String uploadUrl = "http://172.18.1.180:9980/group17/upload"; uploadFile(uploadUrl, path); } }