You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1070 lines
40 KiB
1070 lines
40 KiB
package com.bfd.qanda.utils;
|
|
|
|
import java.io.File;
|
|
import java.io.FileOutputStream;
|
|
import java.io.IOException;
|
|
import java.io.InputStream;
|
|
import java.security.KeyManagementException;
|
|
import java.security.NoSuchAlgorithmException;
|
|
import java.security.cert.CertificateException;
|
|
import java.util.LinkedList;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
|
|
import javax.net.ssl.SSLContext;
|
|
import javax.net.ssl.TrustManager;
|
|
import javax.net.ssl.X509TrustManager;
|
|
|
|
import org.apache.http.HttpEntity;
|
|
import org.apache.http.HttpHost;
|
|
import org.apache.http.HttpResponse;
|
|
import org.apache.http.NameValuePair;
|
|
import org.apache.http.StatusLine;
|
|
import org.apache.http.auth.AuthScope;
|
|
import org.apache.http.auth.UsernamePasswordCredentials;
|
|
import org.apache.http.client.AuthCache;
|
|
import org.apache.http.client.ClientProtocolException;
|
|
import org.apache.http.client.CredentialsProvider;
|
|
import org.apache.http.client.HttpClient;
|
|
import org.apache.http.client.HttpRequestRetryHandler;
|
|
import org.apache.http.client.config.RequestConfig;
|
|
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
|
import org.apache.http.client.methods.CloseableHttpResponse;
|
|
import org.apache.http.client.methods.HttpGet;
|
|
import org.apache.http.client.methods.HttpPost;
|
|
import org.apache.http.client.protocol.HttpClientContext;
|
|
import org.apache.http.config.Registry;
|
|
import org.apache.http.config.RegistryBuilder;
|
|
import org.apache.http.config.SocketConfig;
|
|
import org.apache.http.conn.socket.ConnectionSocketFactory;
|
|
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
|
|
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
|
|
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
|
import org.apache.http.entity.ContentType;
|
|
import org.apache.http.entity.StringEntity;
|
|
import org.apache.http.entity.mime.MultipartEntityBuilder;
|
|
import org.apache.http.impl.auth.BasicScheme;
|
|
import org.apache.http.impl.client.BasicAuthCache;
|
|
import org.apache.http.impl.client.BasicCredentialsProvider;
|
|
import org.apache.http.impl.client.CloseableHttpClient;
|
|
import org.apache.http.impl.client.HttpClientBuilder;
|
|
import org.apache.http.impl.client.HttpClients;
|
|
import org.apache.http.impl.client.LaxRedirectStrategy;
|
|
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
|
import org.apache.http.message.BasicNameValuePair;
|
|
import org.apache.http.util.EntityUtils;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import com.bfd.qanda.entity.Constants;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
* 下载工具类
|
|
* @author jian.mao
|
|
* @date 2023年9月19日
|
|
* @description
|
|
*/
|
|
public class DownLoadUtil {
|
|
|
|
private static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36";
|
|
private final static Logger log = LoggerFactory.getLogger(DownLoadUtil.class);
|
|
/** 代理服务器(产品官网 www.16yun.cn) **/
|
|
final static String PROXYHOST = "u270.40.tp.16yun.cn";
|
|
final static Integer PROXYPORT = 6448;
|
|
/** 代理验证信息 **/
|
|
final static String PROXYUSER = "16HFBVJC";
|
|
final static String PROXYPASS = "897944";
|
|
|
|
private static PoolingHttpClientConnectionManager cm = null;
|
|
private static HttpRequestRetryHandler httpRequestRetryHandler = null;
|
|
private static HttpHost proxy = null;
|
|
|
|
private static CredentialsProvider credsProvider = null;
|
|
private static RequestConfig reqConfig = null;
|
|
|
|
static {
|
|
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory
|
|
.getSocketFactory();
|
|
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory
|
|
.getSocketFactory();
|
|
|
|
Registry registry = RegistryBuilder.create().register("http", plainsf)
|
|
.register("https", sslsf).build();
|
|
|
|
cm = new PoolingHttpClientConnectionManager(registry);
|
|
cm.setMaxTotal(20);
|
|
cm.setDefaultMaxPerRoute(5);
|
|
|
|
proxy = new HttpHost(PROXYHOST, PROXYPORT, "https");
|
|
|
|
credsProvider = new BasicCredentialsProvider();
|
|
credsProvider.setCredentials(AuthScope.ANY,
|
|
new UsernamePasswordCredentials(PROXYUSER, PROXYPASS));
|
|
|
|
reqConfig = RequestConfig.custom().setConnectionRequestTimeout(5000)
|
|
.setConnectTimeout(5000).setSocketTimeout(5000)
|
|
.setExpectContinueEnabled(false)
|
|
.setProxy(new HttpHost(PROXYHOST, PROXYPORT)).build();
|
|
}
|
|
|
|
/**
|
|
* 模拟客户端get请求
|
|
*
|
|
* @param url
|
|
* 模拟请求得url
|
|
* @param headers
|
|
* 头部信息，没有可以不传
|
|
* @return
|
|
*/
|
|
@SafeVarargs
|
|
public static String proxyDoGet(String url, Map<String, Object>... headers) {
|
|
// 设置超时时间
|
|
int timeout = 30;
|
|
RequestConfig config = RequestConfig.custom()
|
|
.setConnectTimeout(timeout * 1000)
|
|
.setConnectionRequestTimeout(timeout * 1000)
|
|
.setSocketTimeout(timeout * 1000).build();
|
|
SocketConfig socketConfig = SocketConfig.custom()
|
|
.setSoKeepAlive(false)
|
|
.setSoLinger(1)
|
|
.setSoReuseAddress(true)
|
|
.setSoTimeout(timeout * 1000)
|
|
.setTcpNoDelay(true).build();
|
|
AuthCache authCache = new BasicAuthCache();
|
|
authCache.put(proxy, new BasicScheme());
|
|
HttpClientContext localContext = HttpClientContext.create();
|
|
localContext.setAuthCache(authCache);
|
|
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
|
|
CloseableHttpClient httpClient = httpBuilder
|
|
.setDefaultSocketConfig(socketConfig)
|
|
.setDefaultRequestConfig(config)
|
|
.setDefaultCredentialsProvider(credsProvider).build();
|
|
HttpGet httpGet = new HttpGet(url);
|
|
httpGet.setConfig(reqConfig);
|
|
if (headers != null && headers.length > 0) {
|
|
Map<String, Object> tempHeaders = headers[0];
|
|
for (String key : tempHeaders.keySet()) {
|
|
httpGet.setHeader(key, tempHeaders.get(key).toString());
|
|
}
|
|
} else {
|
|
httpGet.setHeader("Accept",
|
|
"application/json, text/javascript, */*; q=0.01");
|
|
httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
|
|
}
|
|
CloseableHttpResponse response = null;
|
|
String html = "";
|
|
int notFundCode = 404;
|
|
int successCode = 200;
|
|
try {
|
|
response = httpClient.execute(httpGet, localContext);
|
|
// 从响应模型中获取响应实体
|
|
HttpEntity responseEntity = response.getEntity();
|
|
StatusLine statusLine = response.getStatusLine();
|
|
System.out.println("响应状态为:" + response.getStatusLine());
|
|
if (statusLine.getStatusCode() == successCode) {
|
|
if (responseEntity != null) {
|
|
html = EntityUtils.toString(responseEntity, "utf-8");
|
|
System.out.println("响应内容长度为:"
|
|
+ responseEntity.getContentLength());
|
|
// 下载结果为空不正常
|
|
if (html.equals(Constants.EMPTY)) {
|
|
html = "Download failed error is:reslut is null";
|
|
}
|
|
}
|
|
} else if (statusLine.getStatusCode() == notFundCode) {
|
|
html = "<h2>页面404，正常结束请求即可</h2>";
|
|
} else {
|
|
throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
|
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
html = "Download failed error is:reslut is null";
|
|
}finally{
|
|
try {
|
|
response.close();
|
|
httpClient.close();
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
return html;
|
|
|
|
}
|
|
|
|
|
|
public static String httpsslProxyGet(String url, Map<String, Object>... headers) throws Exception {
|
|
//采用绕过验证的方式处理https请求
|
|
SSLContext sslcontext = createIgnoreVerifySSL();
|
|
|
|
// 设置协议http和https对应的处理socket链接工厂的对象
|
|
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
|
|
.register("http", PlainConnectionSocketFactory.INSTANCE)
|
|
.register("https", new SSLConnectionSocketFactory(sslcontext))
|
|
.build();
|
|
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
|
|
connManager.setMaxTotal(50);
|
|
connManager.setDefaultMaxPerRoute(10);
|
|
HttpClients.custom().setConnectionManager(connManager);
|
|
// 设置超时时间
|
|
int timeout = 30;
|
|
RequestConfig config = RequestConfig.custom()
|
|
.setConnectTimeout(timeout * 1000)
|
|
.setConnectionRequestTimeout(timeout * 1000)
|
|
.setSocketTimeout(timeout * 1000).build();
|
|
SocketConfig socketConfig = SocketConfig.custom()
|
|
.setSoKeepAlive(false)
|
|
.setSoLinger(1)
|
|
.setSoReuseAddress(true)
|
|
.setSoTimeout(timeout * 1000)
|
|
.setTcpNoDelay(true).build();
|
|
AuthCache authCache = new BasicAuthCache();
|
|
authCache.put(proxy, new BasicScheme());
|
|
HttpClientContext localContext = HttpClientContext.create();
|
|
localContext.setAuthCache(authCache);
|
|
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
|
|
CloseableHttpClient httpClient = httpBuilder
|
|
.setConnectionManager(connManager)
|
|
.setDefaultSocketConfig(socketConfig)
|
|
.setDefaultRequestConfig(config)
|
|
.setDefaultCredentialsProvider(credsProvider).build();
|
|
HttpGet httpGet = new HttpGet(url);
|
|
httpGet.setConfig(reqConfig);
|
|
if (headers != null && headers.length > 0) {
|
|
Map<String, Object> tempHeaders = headers[0];
|
|
for (String key : tempHeaders.keySet()) {
|
|
httpGet.setHeader(key, tempHeaders.get(key).toString());
|
|
}
|
|
} else {
|
|
httpGet.setHeader("Accept",
|
|
"application/json, text/javascript, */*; q=0.01");
|
|
httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
|
|
}
|
|
CloseableHttpResponse response = null;
|
|
String html = "";
|
|
int notFundCode = 404;
|
|
int successCode = 200;
|
|
try {
|
|
response = httpClient.execute(httpGet, localContext);
|
|
// 从响应模型中获取响应实体
|
|
HttpEntity responseEntity = response.getEntity();
|
|
StatusLine statusLine = response.getStatusLine();
|
|
System.out.println("响应状态为:" + response.getStatusLine());
|
|
if (statusLine.getStatusCode() == successCode) {
|
|
if (responseEntity != null) {
|
|
html = EntityUtils.toString(responseEntity, "utf-8");
|
|
System.out.println("响应内容长度为:"
|
|
+ responseEntity.getContentLength());
|
|
// 下载结果为空不正常
|
|
if (html.equals(Constants.EMPTY)) {
|
|
html = "Download failed error is:reslut is null";
|
|
}
|
|
}
|
|
} else if (statusLine.getStatusCode() == notFundCode) {
|
|
html = "<h2>页面404，正常结束请求即可</h2>";
|
|
} else {
|
|
throw new Exception("请求错误，code码为：" + statusLine.getStatusCode());
|
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
html = "Download failed error is:reslut is null";
|
|
}finally{
|
|
try {
|
|
response.close();
|
|
httpClient.close();
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
return html;
|
|
|
|
}
|
|
|
|
|
|
/**
|
|
* json参数方式POST提交
|
|
* @param url
|
|
* @param params
|
|
* @return
|
|
*/
|
|
public static String doPost(String url, String params, Map<String, Object>... headers){
|
|
String strResult = "";
|
|
//设置超时时间
|
|
int timeout = 180;
|
|
RequestConfig config = RequestConfig.custom().
|
|
setConnectTimeout(timeout * 1000).
|
|
setConnectionRequestTimeout(timeout * 1000).
|
|
setSocketTimeout(timeout * 1000).build();
|
|
SocketConfig socketConfig = SocketConfig.custom()
|
|
.setSoKeepAlive(false)
|
|
.setSoLinger(1)
|
|
.setSoReuseAddress(true)
|
|
.setSoTimeout(timeout * 1000)
|
|
.setTcpNoDelay(true).build();
|
|
// AuthCache authCache = new BasicAuthCache();
|
|
// authCache.put(proxy, new BasicScheme());
|
|
// HttpClientContext localContext = HttpClientContext.create();
|
|
// localContext.setAuthCache(authCache);
|
|
// 1. 获取默认的client实例
|
|
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
|
|
httpBuilder.setUserAgent(ua);
|
|
HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
|
|
// HttpClient client = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
|
|
// .setDefaultCredentialsProvider(credsProvider).build();
|
|
// 2. 创建httppost实例
|
|
HttpPost httpPost = new HttpPost(url);
|
|
// httpPost.setConfig(reqConfig);
|
|
if (headers != null && headers.length > 0) {
|
|
Map<String, Object> tempHeaders = headers[0];
|
|
for (String key : tempHeaders.keySet()) {
|
|
httpPost.setHeader(key, tempHeaders.get(key).toString());
|
|
}
|
|
} else {
|
|
httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
|
|
}
|
|
HttpResponse resp = null;
|
|
try {
|
|
httpPost.setEntity(new StringEntity(params,"utf-8"));
|
|
resp = client.execute(httpPost);
|
|
// resp = client.execute(httpPost,localContext);
|
|
StatusLine statusLine = resp.getStatusLine();
|
|
System.out.println("响应状态为:" + resp.getStatusLine());
|
|
int notFundCode = 404;
|
|
int successCode = 200;
|
|
if(statusLine.getStatusCode() == successCode){
|
|
// 7. 获取响应entity
|
|
HttpEntity respEntity = resp.getEntity();
|
|
strResult = EntityUtils.toString(respEntity, "UTF-8");
|
|
if(strResult.equals(Constants.EMPTY)){
|
|
strResult = "Download failed error is:reslut is null";
|
|
}
|
|
}else{
|
|
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
|
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
strResult = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
|
|
}
|
|
return strResult;
|
|
}
|
|
public static String httpPost(String url,String params) {
|
|
String html="";
|
|
html = doPost(url,params);
|
|
int i = 1;
|
|
while(true){
|
|
if(html.contains("Download failed error is:")){
|
|
log.error("DownLoadUtil------------->download is failure,url is:"+url);
|
|
DateUtil.sleep(5000);
|
|
i++;
|
|
}else{
|
|
break;
|
|
}
|
|
if(i > 5){
|
|
break;
|
|
}
|
|
html = doPost(url,params);
|
|
}
|
|
return html;
|
|
}
|
|
/**
|
|
* 绕过验证
|
|
*
|
|
* @return
|
|
* @throws NoSuchAlgorithmException
|
|
* @throws KeyManagementException
|
|
*/
|
|
public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
|
|
SSLContext sc = SSLContext.getInstance("SSLv3");
|
|
|
|
// 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
|
|
X509TrustManager trustManager = new X509TrustManager() {
|
|
@Override
|
|
public void checkClientTrusted(
|
|
java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
|
|
String paramString) throws CertificateException {
|
|
}
|
|
|
|
@Override
|
|
public void checkServerTrusted(
|
|
java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
|
|
String paramString) throws CertificateException {
|
|
}
|
|
|
|
@Override
|
|
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
|
|
return null;
|
|
}
|
|
};
|
|
|
|
sc.init(null, new TrustManager[] { trustManager }, null);
|
|
return sc;
|
|
}
|
|
/**
|
|
* 模拟请求
|
|
*
|
|
* @param url 资源地址
|
|
* @param map 参数列表
|
|
* @param encoding 编码
|
|
* @return
|
|
* @throws NoSuchAlgorithmException
|
|
* @throws KeyManagementException
|
|
* @throws IOException
|
|
* @throws ClientProtocolException
|
|
*/
|
|
public static String httpsslGet(String url,Map<String, Object> ... headers) {
|
|
String html="";
|
|
CloseableHttpClient client = null;
|
|
HttpEntity responseEntity = null;
|
|
CloseableHttpResponse response = null;
|
|
try {
|
|
log.debug("DownLoadUtil------------->设置下载相关信息， start....");
|
|
//采用绕过验证的方式处理https请求
|
|
SSLContext sslcontext = createIgnoreVerifySSL();
|
|
|
|
// 设置协议http和https对应的处理socket链接工厂的对象
|
|
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
|
|
.register("http", PlainConnectionSocketFactory.INSTANCE)
|
|
.register("https", new SSLConnectionSocketFactory(sslcontext))
|
|
.build();
|
|
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
|
|
connManager.setMaxTotal(50);
|
|
connManager.setDefaultMaxPerRoute(10);
|
|
HttpClients.custom().setConnectionManager(connManager);
|
|
//设置超时时间
|
|
int timeout = 30;
|
|
RequestConfig config = RequestConfig.custom().
|
|
setConnectTimeout(timeout * 1000).
|
|
setConnectionRequestTimeout(timeout * 1000).
|
|
setSocketTimeout(timeout * 1000).build();
|
|
SocketConfig socketConfig = SocketConfig.custom()
|
|
.setSoKeepAlive(false)
|
|
.setSoLinger(1)
|
|
.setSoReuseAddress(true)
|
|
.setSoTimeout(10000)
|
|
.setTcpNoDelay(true).build();
|
|
// 设置重定向策略
|
|
LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy();
|
|
//创建自定义的httpclient对象
|
|
client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setRedirectStrategy(redirectStrategy).setDefaultSocketConfig(socketConfig).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36").build();
|
|
// CloseableHttpClient client = HttpClients.createDefault();
|
|
|
|
HttpGet httpGet = new HttpGet(url);
|
|
if(headers != null && headers.length > 0){
|
|
Map<String, Object> tempHeaders = headers[0];
|
|
for (String key : tempHeaders.keySet()) {
|
|
httpGet.setHeader(key,tempHeaders.get(key).toString());
|
|
}
|
|
}else{
|
|
httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
|
|
httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
|
|
httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
|
|
}
|
|
log.debug("DownLoadUtil------------->设置下载相关信息， end....");
|
|
try {
|
|
int notFundCode = 404;
|
|
int successCode = 200;
|
|
log.debug("DownLoadUtil------------->下载执行，start....");
|
|
httpGet.setConfig(config);
|
|
response = client.execute(httpGet);
|
|
log.debug("DownLoadUtil------------->下载执行，end....");
|
|
// 从响应模型中获取响应实体
|
|
StatusLine statusLine = response.getStatusLine();
|
|
log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is start ....");
|
|
System.out.println("响应状态为:" + response.getStatusLine());
|
|
responseEntity = response.getEntity();
|
|
log.debug("DownLoadUtil------------->响应状态为:" + response.getStatusLine()+",下载请求没问题url:"+url+"，read is end ....");
|
|
if(statusLine.getStatusCode() == successCode){
|
|
if (responseEntity != null) {
|
|
html=EntityUtils.toString(responseEntity,"utf-8");
|
|
System.out.println("响应内容长度为:" + responseEntity.getContentLength());
|
|
}
|
|
}else if(statusLine.getStatusCode() == notFundCode){
|
|
html = "<h2>页面404，正常结束请求即可</h2>";
|
|
}else{
|
|
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
|
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
|
|
|
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
|
|
}finally{
|
|
try {
|
|
responseEntity.getContent().close();
|
|
response.close();
|
|
client.close();
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
}
|
|
|
|
|
|
return html;
|
|
}
|
|
|
|
public static String httpSSLGet(String url,Map<String, Object> ... headers) {
|
|
String html="";
|
|
html = httpsslGet(url,headers);
|
|
int i = 1;
|
|
while(true){
|
|
if(html.contains("Download failed error is:")){
|
|
log.error("DownLoadUtil------------->download is failure,url is:"+url);
|
|
DateUtil.sleep(30000);
|
|
i++;
|
|
}else{
|
|
break;
|
|
}
|
|
if(i > 5){
|
|
break;
|
|
}
|
|
html = httpsslGet(url,headers);
|
|
}
|
|
return html;
|
|
}
|
|
public static String doPostFrom(String url,Map<String, Object> param,Map<String, Object> ... headers){
|
|
//设置超时时间
|
|
int timeout = 15;
|
|
RequestConfig config = RequestConfig.custom().
|
|
setConnectTimeout(timeout * 1000).
|
|
setConnectionRequestTimeout(timeout * 1000).
|
|
setSocketTimeout(timeout * 1000).build();
|
|
SocketConfig socketConfig = SocketConfig.custom()
|
|
.setSoKeepAlive(false)
|
|
.setSoLinger(1)
|
|
.setSoReuseAddress(true)
|
|
.setSoTimeout(10000)
|
|
.setTcpNoDelay(true).build();
|
|
// AuthCache authCache = new BasicAuthCache();
|
|
// authCache.put(proxy, new BasicScheme());
|
|
// HttpClientContext localContext = HttpClientContext.create();
|
|
// localContext.setAuthCache(authCache);
|
|
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
|
|
httpBuilder.setUserAgent(ua);
|
|
// HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).setConnectionManager(cm)
|
|
// .setDefaultCredentialsProvider(credsProvider).build();
|
|
HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
|
|
HttpPost httpPost = new HttpPost(url);
|
|
// httpPost.setConfig(reqConfig);
|
|
if(headers != null && headers.length > 0){
|
|
Map<String, Object> tempHeaders = headers[0];
|
|
for (String key : tempHeaders.keySet()) {
|
|
httpPost.setHeader(key,tempHeaders.get(key).toString());
|
|
}
|
|
}else{
|
|
httpPost.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
|
|
httpPost.addHeader("accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
|
|
httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
|
|
httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36");
|
|
// httpPost.addHeader("Referer", "http://www.neeq.com.cn/rule/Business_rules.html");
|
|
}
|
|
// 创建请求参数
|
|
List<NameValuePair> list = new LinkedList<>();
|
|
for (String key : param.keySet()) {
|
|
BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
|
|
list.add(param1);
|
|
}
|
|
// 使用URL实体转换工具
|
|
String html="";
|
|
try {
|
|
UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
|
|
httpPost.setEntity(entityParam);
|
|
HttpResponse response = httpClient.execute(httpPost);
|
|
// HttpResponse response = httpClient.execute(httpPost,localContext);
|
|
// 从响应模型中获取响应实体
|
|
HttpEntity responseEntity = response.getEntity();
|
|
StatusLine statusLine = response.getStatusLine();
|
|
System.out.println("响应状态为:" + response.getStatusLine());
|
|
int notFundCode = 404;
|
|
int successCode = 200;
|
|
if(statusLine.getStatusCode() == successCode){
|
|
if (responseEntity != null) {
|
|
html=EntityUtils.toString(responseEntity,"utf-8");
|
|
}
|
|
}else{
|
|
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
|
|
}
|
|
|
|
return html;
|
|
|
|
}
|
|
public static String httpPostForm(String url,Map<String,Object> params,Map<String, Object> ... headers) {
|
|
String html="";
|
|
html = doPostFrom(url,params);
|
|
int i = 1;
|
|
while(true){
|
|
if(html.contains("Download failed error is:")){
|
|
log.error("DownLoadUtil------------->download is failure,url is:"+url);
|
|
DateUtil.sleep(5000);
|
|
i++;
|
|
}else{
|
|
break;
|
|
}
|
|
if(i > 5){
|
|
break;
|
|
}
|
|
html = doPostFrom(url,params,headers);
|
|
}
|
|
return html;
|
|
}
|
|
|
|
public static String dosslPost(String url,String params,Map<String, Object> ... headers) {
|
|
String html="";
|
|
CloseableHttpClient client = null;
|
|
HttpEntity responseEntity = null;
|
|
CloseableHttpResponse response = null;
|
|
try {
|
|
//采用绕过验证的方式处理https请求
|
|
SSLContext sslcontext = createIgnoreVerifySSL();
|
|
// 设置协议http和https对应的处理socket链接工厂的对象
|
|
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
|
|
.register("http", PlainConnectionSocketFactory.INSTANCE)
|
|
.register("https", new SSLConnectionSocketFactory(sslcontext))
|
|
.build();
|
|
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
|
|
HttpClients.custom().setConnectionManager(connManager);
|
|
//设置超时时间
|
|
int timeout = 5;
|
|
RequestConfig config = RequestConfig.custom().
|
|
setConnectTimeout(timeout * 1000).
|
|
setConnectionRequestTimeout(timeout * 1000).
|
|
setSocketTimeout(timeout * 1000).build();
|
|
SocketConfig socketConfig = SocketConfig.custom()
|
|
.setSoKeepAlive(false)
|
|
.setSoLinger(1)
|
|
.setSoReuseAddress(true)
|
|
.setSoTimeout(10000)
|
|
.setTcpNoDelay(true).build();
|
|
//创建自定义的httpclient对象
|
|
client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
|
|
// CloseableHttpClient client = HttpClients.createDefault();
|
|
// 2. 创建httppost实例
|
|
HttpPost httpPost = new HttpPost(url);
|
|
// httpPost.setConfig(reqConfig);
|
|
httpPost.addHeader("Content-Type", "application/json;charset=utf-8");
|
|
if(headers != null && headers.length > 0){
|
|
Map<String, Object> tempHeaders = headers[0];
|
|
for (String key : tempHeaders.keySet()) {
|
|
httpPost.setHeader(key,tempHeaders.get(key).toString());
|
|
}
|
|
}else{
|
|
httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
|
|
httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
|
|
httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
|
|
}
|
|
|
|
try {
|
|
httpPost.setEntity(new StringEntity(params,"utf-8"));
|
|
response = client.execute(httpPost);
|
|
int notFundCode = 404;
|
|
int successCode = 200;
|
|
// 从响应模型中获取响应实体
|
|
StatusLine statusLine = response.getStatusLine();
|
|
System.out.println("响应状态为:" + response.getStatusLine());
|
|
responseEntity = response.getEntity();
|
|
if(statusLine.getStatusCode() == successCode){
|
|
if (responseEntity != null) {
|
|
html=EntityUtils.toString(responseEntity,"utf-8");
|
|
System.out.println("响应内容长度为:" + responseEntity.getContentLength());
|
|
}
|
|
}else if(statusLine.getStatusCode() == notFundCode){
|
|
html = "<h2>页面404，正常结束请求即可</h2>";
|
|
}else{
|
|
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
|
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
|
|
|
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
|
|
}finally{
|
|
try {
|
|
responseEntity.getContent().close();
|
|
response.close();
|
|
client.close();
|
|
} catch (UnsupportedOperationException e) {
|
|
e.printStackTrace();
|
|
} catch (IOException e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
}
|
|
|
|
|
|
return html;
|
|
}
|
|
public static String dosslPostForm(String url,Map<String, Object> param,Map<String, Object> ... headers) {
|
|
String html="";
|
|
try {
|
|
//采用绕过验证的方式处理https请求
|
|
SSLContext sslcontext = createIgnoreVerifySSL();
|
|
|
|
// 设置协议http和https对应的处理socket链接工厂的对象
|
|
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
|
|
.register("http", PlainConnectionSocketFactory.INSTANCE)
|
|
.register("https", new SSLConnectionSocketFactory(sslcontext))
|
|
.build();
|
|
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
|
|
HttpClients.custom().setConnectionManager(connManager);
|
|
//设置超时时间
|
|
int timeout = 5;
|
|
RequestConfig config = RequestConfig.custom().
|
|
setConnectTimeout(timeout * 1000).
|
|
setConnectionRequestTimeout(timeout * 1000).
|
|
setSocketTimeout(timeout * 1000).build();
|
|
SocketConfig socketConfig = SocketConfig.custom()
|
|
.setSoKeepAlive(false)
|
|
.setSoLinger(1)
|
|
.setSoReuseAddress(true)
|
|
.setSoTimeout(10000)
|
|
.setTcpNoDelay(true).build();
|
|
//创建自定义的httpclient对象
|
|
CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).setDefaultSocketConfig(socketConfig).build();
|
|
// CloseableHttpClient client = HttpClients.createDefault();
|
|
// 2. 创建httppost实例
|
|
HttpPost httpPost = new HttpPost(url);
|
|
// httpPost.setConfig(reqConfig);
|
|
if(headers != null && headers.length > 0){
|
|
Map<String, Object> tempHeaders = headers[0];
|
|
for (String key : tempHeaders.keySet()) {
|
|
httpPost.setHeader(key,tempHeaders.get(key).toString());
|
|
}
|
|
}else{
|
|
httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
|
|
httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
|
|
httpPost.addHeader("content-type", "application/x-www-form-urlencoded");
|
|
httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
|
|
}
|
|
|
|
// 创建请求参数
|
|
List<NameValuePair> list = new LinkedList<>();
|
|
for (String key : param.keySet()) {
|
|
BasicNameValuePair param1 = new BasicNameValuePair(key,param.get(key).toString());
|
|
list.add(param1);
|
|
}
|
|
// 使用URL实体转换工具
|
|
try {
|
|
UrlEncodedFormEntity entityParam = new UrlEncodedFormEntity(list, "UTF-8");
|
|
httpPost.setEntity(entityParam);
|
|
HttpResponse response = client.execute(httpPost);
|
|
// HttpResponse response = httpClient.execute(httpPost,localContext);
|
|
// 从响应模型中获取响应实体
|
|
int notFundCode = 404;
|
|
int successCode = 200;
|
|
HttpEntity responseEntity = response.getEntity();
|
|
StatusLine statusLine = response.getStatusLine();
|
|
System.out.println("响应状态为:" + response.getStatusLine());
|
|
if(statusLine.getStatusCode() == successCode){
|
|
if (responseEntity != null) {
|
|
html=EntityUtils.toString(responseEntity,"utf-8");
|
|
}
|
|
}else{
|
|
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
|
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
|
|
}
|
|
|
|
|
|
return html;
|
|
}
|
|
public static String httpSSLPostForm(String url,Map<String, Object> params,Map<String, Object> ...headers) {
|
|
String html="";
|
|
try {
|
|
html = dosslPostForm(url,params,headers);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
// TODO: handle exception
|
|
html = "Download failed error is:Exception!";
|
|
}
|
|
int i = 1;
|
|
while(true){
|
|
if(html.contains("Download failed error is:")){
|
|
log.error("DownLoadUtil------------->download is failure,url is:"+url);
|
|
DateUtil.sleep(30000);
|
|
i++;
|
|
}else{
|
|
break;
|
|
}
|
|
if(i > 5){
|
|
break;
|
|
}
|
|
try {
|
|
html = dosslPostForm(url,params,headers);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
// TODO: handle exception
|
|
html = "Download failed error is:Exception!";
|
|
}
|
|
}
|
|
return html;
|
|
}
|
|
public static String httpSSLPost(String url,String params,Map<String, Object> ...headers) {
|
|
String html="";
|
|
try {
|
|
html = dosslPost(url,params,headers);
|
|
} catch (Throwable e) {
|
|
e.printStackTrace();
|
|
// TODO: handle exception
|
|
html = "Download failed error is:Exception!";
|
|
}
|
|
int i = 1;
|
|
while(true){
|
|
if(html.contains("Download failed error is:")){
|
|
log.error("DownLoadUtil------------->download is failure,url is:"+url);
|
|
DateUtil.sleep(30000);
|
|
i++;
|
|
}else{
|
|
break;
|
|
}
|
|
if(i > 5){
|
|
break;
|
|
}
|
|
try {
|
|
html = dosslPost(url,params,headers);
|
|
} catch (Throwable e) {
|
|
e.printStackTrace();
|
|
// TODO: handle exception
|
|
html = "Download failed error is:Exception!";
|
|
}
|
|
}
|
|
return html;
|
|
}
|
|
|
|
/**
|
|
* 模拟客户端get请求
|
|
* @param url 模拟请求得url
|
|
* @param headers 头部信息，没有可以不传
|
|
* @return
|
|
*/
|
|
public static String doGet(String url,Map<String, Object> ... headers){
|
|
//设置超时时间
|
|
int timeout = 15;
|
|
RequestConfig config = RequestConfig.custom().
|
|
setConnectTimeout(timeout * 1000).
|
|
setConnectionRequestTimeout(timeout * 1000).
|
|
setSocketTimeout(timeout * 1000).build();
|
|
SocketConfig socketConfig = SocketConfig.custom()
|
|
.setSoKeepAlive(false)
|
|
.setSoLinger(1)
|
|
.setSoReuseAddress(true)
|
|
.setSoTimeout(10000)
|
|
.setTcpNoDelay(true).build();
|
|
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
|
|
httpBuilder.setUserAgent(ua);
|
|
HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
|
|
HttpGet httpGet = new HttpGet(url);
|
|
if(headers != null && headers.length > 0){
|
|
Map<String, Object> tempHeaders = headers[0];
|
|
for (String key : tempHeaders.keySet()) {
|
|
httpGet.setHeader(key,tempHeaders.get(key).toString());
|
|
}
|
|
}else{
|
|
httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
|
|
httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
|
|
}
|
|
String html="";
|
|
try {
|
|
int notFundCode = 404;
|
|
int successCode = 200;
|
|
HttpResponse response = httpClient.execute(httpGet);
|
|
// 从响应模型中获取响应实体
|
|
HttpEntity responseEntity = response.getEntity();
|
|
StatusLine statusLine = response.getStatusLine();
|
|
System.out.println("响应状态为:" + response.getStatusLine());
|
|
if(statusLine.getStatusCode() == successCode){
|
|
if (responseEntity != null) {
|
|
html=EntityUtils.toString(responseEntity,"utf-8");
|
|
if(html.equals("")){
|
|
html = "Download failed error is:reslut is null";
|
|
}
|
|
}
|
|
}else if(statusLine.getStatusCode() == notFundCode){
|
|
html = "<h2>页面404，正常结束请求即可</h2>";
|
|
}else{
|
|
throw new Exception("请求错误，code码为："+statusLine.getStatusCode());
|
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
html = "Download failed error is:"+ThrowMessageUtil.getErrmessage(e);
|
|
}
|
|
return html;
|
|
|
|
}
|
|
|
|
/**
|
|
* 文件下载
|
|
* @param fileURL 文件连接
|
|
* @param saveFilePath 文件存储地址
|
|
* @throws IOException
|
|
*/
|
|
public static void downloadFile(String fileURL, String saveFilePath,Map<String, Object> ... headers) {
|
|
//设置超时时间
|
|
int timeout = 60;
|
|
RequestConfig config = RequestConfig.custom().
|
|
setConnectTimeout(timeout * 1000).
|
|
setConnectionRequestTimeout(timeout * 1000).
|
|
setSocketTimeout(timeout * 1000).build();
|
|
SocketConfig socketConfig = SocketConfig.custom()
|
|
.setSoKeepAlive(false)
|
|
.setSoLinger(1)
|
|
.setSoReuseAddress(true)
|
|
.setSoTimeout(timeout * 1000)
|
|
.setTcpNoDelay(true).build();
|
|
HttpClientBuilder httpBuilder = HttpClientBuilder.create();
|
|
httpBuilder.setUserAgent(ua);
|
|
HttpClient httpClient = httpBuilder.setDefaultSocketConfig(socketConfig).setDefaultRequestConfig(config).build();
|
|
HttpGet httpGet = new HttpGet(fileURL);
|
|
if(headers != null && headers.length > 0){
|
|
Map<String, Object> tempHeaders = headers[0];
|
|
for (String key : tempHeaders.keySet()) {
|
|
httpGet.setHeader(key,tempHeaders.get(key).toString());
|
|
}
|
|
}else{
|
|
httpGet.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
|
|
httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
|
|
}
|
|
try {
|
|
int successCode = 200;
|
|
HttpResponse response = httpClient.execute(httpGet);
|
|
// 从响应模型中获取响应实体
|
|
StatusLine statusLine = response.getStatusLine();
|
|
int statusCode = statusLine.getStatusCode();
|
|
if(statusCode == successCode){
|
|
// 获取响应体
|
|
HttpEntity entity = response.getEntity();
|
|
if (entity != null) {
|
|
// 创建输入流来读取文件内容
|
|
try (InputStream inputStream = entity.getContent();
|
|
FileOutputStream fileOutputStream = new FileOutputStream(new File(saveFilePath))) {
|
|
|
|
byte[] buffer = new byte[4096];
|
|
int bytesRead;
|
|
while ((bytesRead = inputStream.read(buffer)) != -1) {
|
|
fileOutputStream.write(buffer, 0, bytesRead);
|
|
}
|
|
}
|
|
log.info("文件下载成功，保存路径为: {}" , saveFilePath);
|
|
}
|
|
} else {
|
|
log.error("文件下载失败，HTTP 响应状态码为: {}" , statusCode);
|
|
}
|
|
EntityUtils.consume(response.getEntity());
|
|
} catch (Exception e) {
|
|
log.error("下载文件异常：{}",e);
|
|
}
|
|
|
|
}
|
|
|
|
/**
|
|
* 删除文件
|
|
* @param filePath
|
|
*/
|
|
public static void delFile(String filePath) {
|
|
try {
|
|
// 创建 File 对象
|
|
File file = new File(filePath);
|
|
// 检查文件是否存在
|
|
if (file.exists()) {
|
|
// 尝试删除文件
|
|
if (file.delete()) {
|
|
log.info("文件删除成功: " + filePath);
|
|
} else {
|
|
log.error("无法删除文件: " + filePath);
|
|
}
|
|
} else {
|
|
log.warn("文件不存在: " + filePath);
|
|
}
|
|
} catch (Exception e) {
|
|
// TODO: handle exception
|
|
log.error("删除文件异常：{}",filePath);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param uploadUrl
|
|
* @param filePath
|
|
*/
|
|
public static String uploadFile(String uploadUrl, String filePath) {
|
|
// 创建HttpClient实例
|
|
CloseableHttpClient httpClient = HttpClients.createDefault();
|
|
String gofastUrl = "";
|
|
try {
|
|
// 创建HttpPost请求
|
|
HttpPost httpPost = new HttpPost(uploadUrl);
|
|
// 要上传的文件
|
|
File file = new File(filePath);
|
|
if (!file.exists()) {
|
|
log.error("文件不存在：{}" , filePath);
|
|
return gofastUrl;
|
|
}
|
|
// 构建文件上传请求体
|
|
HttpEntity entity = MultipartEntityBuilder.create()
|
|
.addBinaryBody("file", file, ContentType.DEFAULT_BINARY, file.getName())
|
|
.build();
|
|
|
|
// 设置请求体
|
|
httpPost.setEntity(entity);
|
|
|
|
// 执行请求并获取响应
|
|
CloseableHttpResponse response = httpClient.execute(httpPost);
|
|
try {
|
|
int statusCode = response.getStatusLine().getStatusCode();
|
|
int successCode = 200;
|
|
if (statusCode == successCode) {
|
|
// 解析响应内容
|
|
gofastUrl = EntityUtils.toString(response.getEntity());
|
|
log.info("文件上传成功，响应：{}" , gofastUrl);
|
|
} else {
|
|
log.error("文件上传失败，HTTP 响应码：{}" , statusCode);
|
|
}
|
|
}catch (Exception e) {
|
|
// TODO: handle exception
|
|
} finally {
|
|
try {
|
|
response.close();
|
|
} catch (IOException e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
}catch(Exception e){
|
|
e.printStackTrace();
|
|
} finally {
|
|
try {
|
|
httpClient.close();
|
|
} catch (IOException e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
return gofastUrl;
|
|
}
|
|
public static void main(String[] args) throws Exception {
|
|
// String url = "https://public-cdn-video-data-algeng.oss-cn-wulanchabu.aliyuncs.com/inference_output%2Fvideo%2F2024-09-14%2Fa13bd277-691e-4a3e-9096-ec5e9a449aa3%2Foutput.mp4?Expires=1726313297&OSSAccessKeyId=LTAI5tGLnRTkBjLuYPjNcKQ8&Signature=TmZDbYyisMoNi0smdEUCkzooTZI%3D";
|
|
String path = "D:\\123.mp4";
|
|
// downloadFile(url,path);
|
|
String uploadUrl = "http://172.18.1.180:9980/group17/upload";
|
|
uploadFile(uploadUrl, path);
|
|
}
|
|
}
|