httpClient使用小结

我不是女神ヾ 2022-05-25 21:07 107阅读 0赞

*  HttpClient 发送请求带json参数：

import com.alibaba.fastjson.JSONObject;
    import com.credit.service.KeepCookieSessionService;
    import org.apache.http.HttpEntity;
    import org.apache.http.client.ClientProtocolException;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpPost;
    import org.apache.http.entity.StringEntity;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.util.EntityUtils;
    import org.springframework.stereotype.Service;
    
    import javax.servlet.http.HttpSession;
    import java.io.IOException;
    import java.io.UnsupportedEncodingException;
    import java.util.HashMap;
    import java.util.Map;
    
    /** * @author susq * @since 2018-04-10-19:04 */
    @Service
    public class App { 
    
        private static void queryFptz() {
            // json键值对的参数都在这里设置
            Map<String, String> map = new HashMap<>();
            map.put("pageSize", "5");
    
            // 创建httppost
            HttpPost httppost = new HttpPost("http://10.199.129.8:10000/zzswy/api/zzs/xxgl/fptz/queryFptz");
            // 创建参数队列
            StringEntity s = null;
            try {
                s = new StringEntity(JSONObject.toJSONString(map));
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            s.setContentEncoding("UTF-8");
            s.setContentType("application/json"); //设置contentType
            CloseableHttpClient httpclient = HttpClients.createDefault();
            try {
                httppost.setEntity(s);
                CloseableHttpResponse response = httpclient.execute(httppost);
                String cookies = response.getFirstHeader("Set-Cookie").getValue();
                try {
                    String responseStr = EntityUtils.toString(response.getEntity(), "UTF-8"));
                } finally {
                    response.close();
                }
            } catch (ClientProtocolException e) {
                e.printStackTrace();
            } catch (UnsupportedEncodingException e1) {
                e1.printStackTrace();
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                // 关闭连接,释放资源
                try {
                    httpclient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    
    }

*  发送application/x-www-form-urlencoded 编码的数据, 只需改一下中间创建和添加entity的部分

List<NameValuePair> nameValuePairList = new ArrayList<>();
    nameValuePairList.add(new BasicNameValuePair("username",username);
    nameValuePairList.add(new BasicNameValuePair("password",password);
    try {
        UrlEncodedFormEntity entity = new UrlEncodedFormEntity(nameValuePairList, "UTF-8");
        HttpPost httppost = new HttpPost(Url);
        httppost.setEntity(entity);
        CloseableHttpResponse response = httpclient.execute(httppost);
        String res = EntityUtils.toString(response.getEntity());
    } catch (UnsupportedEncodingException e) {
        return new DzswjAccountLoginResult();
    }

*  封装一下httpClient, 可以加上连接池配置，get方法，post方法, 发送json, 或者url编码的参数 都可以在里面做一层封装，使用起来更方便。这里我没有详细的去写。  
    使用httpClient连接池，提升性能，可以设置最大连接数，每个路由的连接数，请求超时时间，读取超时时间等。这里的连接池我理解是同一个CloseableHttpClient 内部，可以建立多少个http连接，使用的是同一个CloseableHttpClient 实例。

package com.susq.httpclient;
    
    import org.apache.http.HttpEntity;
    import org.apache.http.client.config.RequestConfig;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpPost;
    import org.apache.http.client.methods.HttpUriRequest;
    import org.apache.http.entity.ContentType;
    import org.apache.http.entity.StringEntity;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClientBuilder;
    import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
    import org.apache.http.util.EntityUtils;
    
    import java.io.IOException;
    import java.util.Map;
    import java.util.Set;
    
    public class HttpTransport { 
    
        private CloseableHttpClient httpClient;
    
        private PoolingHttpClientConnectionManager connMgr;
    
        private RequestConfig requestConfig;
    
        // 链接超时时间（单位毫秒）
        private int conTimeout = 1000;
        // 读取超时时间（单位毫秒）
        private int readTimeOut = 1000;
        // 最大连接数
        private int maxTotal = 2;
        // 同一路由最大连接数
        private int maxPerRoute = 2;
    
        // 如果使用spring配置这个bean,指定init方法就可以，不用构造方法
    // public HttpTransport() { 
    // if (httpClient == null) { 
    // init();
    // }
    // }
    
        public void init() {
            if (httpClient != null) {
                return;
            }
            HttpClientBuilder clientBuilder = HttpClientBuilder.create();
            if (connMgr != null) {
                clientBuilder.setConnectionManager(connMgr);
                clientBuilder.setConnectionManagerShared(true);
                clientBuilder.disableCookieManagement();
            } else {
                if (maxTotal > 0) {
                    clientBuilder.setMaxConnPerRoute(maxPerRoute);
                }
                if (maxPerRoute > 0) {
                    clientBuilder.setMaxConnTotal(maxTotal);
                }
            }
            if (conTimeout > 0 || readTimeOut > 0) {
                RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
                if (conTimeout > 0) {
                    requestConfigBuilder.setConnectTimeout(conTimeout);
                }
                if (readTimeOut > 0) {
                    requestConfigBuilder.setSocketTimeout(readTimeOut);
                }
                requestConfig = requestConfigBuilder.build();
            }
            httpClient = clientBuilder.build();
        }
    
        public String doCommonHttpsPost(String apiUrl, Map<String, String> headers, Object json, String charset, String miniType)
                throws IOException {
            if (null == apiUrl)
                return "";
            if (null == json) {
                json = "{}";
            }
            CloseableHttpResponse response = null;
            try {
                HttpPost httpPost = new HttpPost(apiUrl);
                setHeaders(headers, httpPost);
                httpPost.setConfig(requestConfig);
    
                StringEntity stringEntity = new StringEntity(json.toString(),
                        ContentType.create(miniType, charset));
                stringEntity.setContentEncoding(charset);
                httpPost.setEntity(stringEntity);
                response = httpClient.execute(httpPost);
                HttpEntity entity = response.getEntity();
                return EntityUtils.toString(entity, charset);
            } finally {
                if (response != null) {
                    try {
                        EntityUtils.consume(response.getEntity());
                    } catch (IOException e) {
    
                    }
                }
            }
        }
    
        protected void setHeaders(Map<String, String> headers, HttpUriRequest request) {
            if (headers != null && !headers.isEmpty()) {
                Set<Map.Entry<String, String>> headerSet = headers.entrySet();
                for (Map.Entry<String, String> one : headerSet) {
                    request.addHeader(one.getKey(), one.getValue());
                }
            }
        }
    }

*  获取cookies
    
    创建httpClient的时候，新建一个BasicCookieStore对象，设进去，以后就可以通过这个对象记录cookies的状态了。
    
        HttpClientBuilder clientBuilder = HttpClients.custom();
        CookieStore cookieStore = new BasicCookieStore();
        clientBuilder.setDefaultCookieStore(cookieStore);
 *  保持cookies（爬网站）
    
    httpClient 默认是保持cookie的，除非禁止了cookie : clientBuilder.disableCookieManagement(); 爬网站的时候，连续的接口请求中使用同一个httpClient ，就可以默认带上上次请求后的cookies. 完全类似于打开一个浏览器操作网站。but，如果是并发请求，而不是一个单线程的任务在哪里跑，cookies就会串掉，可能会影响后续的请求。而且CloseableHttpClient 不支持删除或者修改cookies。 所以如果是并发，还是给每个任务创建一个cilent, 属于这个任务里面的一系列请求，可以使用同一个client实例。
 *  遇到的问题

1、 Connection pool shut down httpclient

我遇到的时候，是因为使用连接管理配置没有设置共享，设置以后就好了，clientBuilder.setConnectionManagerShared(true);

2、 response 每次使用完必须关闭释放资源，否则占用线程，线程池不够的时候会一直卡死。

*  翻译的部分PoolingHttpClientConnectionManager的文档，但是感觉收获不大，不继续翻译了

> ClientConnectionPoolManager maintains a pool of HttpClientConnections and is able to service connection requests from multiple execution threads. Connections are pooled on a per route basis. A request for a route which already the manager has persistent connections for available in the pool will be services by leasing a connection from the pool rather than creating a brand new connection.

ClientConnectionPoolManager维护一个HttpClientConnections池，这个连接池能够从多个执行线程中服务连接请求。连接在每个路由的基础上被合并池化。对于已经在池中可用的永久性连接的路由请求，将通过从池中租借连接而不是创建一个全新的连接来提供服务。

> ClientConnectionPoolManager maintains a maximum limit of connection on a per route basis and in total. Per default this implementation will create no more than than 2 concurrent connections per given route and no more 20 connections in total. For many real-world applications these limits may prove too constraining, especially if they use HTTP as a transport protocol for their services. Connection limits, however, can be adjusted using ConnPoolControl methods.

ClientConnectionPoolManager分别在每个路由上和总数上维护着最大的连接限制。默认情况下，这个实现将在每个给定的路由上创建不超过2个并发连接，所有路由总共不超过20个连接。对于许多实际应用程序来说，这些限制可能被证明太过约束，特别是使用HTTP作为其服务的传输协议的。然而，连接限制可以使用ConnPoolControl方法进行调整。

> Total time to live (TTL) set at construction time defines maximum life span of persistent connections regardless of their expiration setting. No persistent connection will be re-used past its TTL value.

在构建时设置的总生存时间 （TTL）定义了持久连接的最大生命周期，而不用考虑它们的过期时间设置。持久连接不会被重新使用，通过它的过去的TTL值。

> The handling of stale connections was changed in version 4.4. Previously, the code would check every connection by default before re-using it. The code now only checks the connection if the elapsed time since the last use of the connection exceeds the timeout that has been set. The default timeout is set to 2000ms

在版本4.4中改变了陈旧连接的处理方式。在此之前，代码将在重新使用之前检查每个连接。代码现在只检查从最后一次使用到现在的时间是否超过了设定的超时时间。默认的超时设置为2000毫秒。