Java基础：利用HttpClient获取网页内容

客官°小女子只卖身不卖艺 2021-09-10 05:28 342阅读 0赞

<table style="BORDER-TOP: #ffffff 17px solid"> 
 <tbody> 
  <tr> 
   <td align="center"> <h1>Java基础：利用HttpClient获取网页内容 </h1> </td> 
  </tr> 
 </tbody> 
</table>

<table style="MARGIN-BOTTOM: 20px"> 
 <tbody> 
  <tr> 
   <td align="center"><span></span><span><font>发布时间：2006.04.24 05:18</font></span>&nbsp;&nbsp;&nbsp;&nbsp;<span><font> 来源：未知</font></span>&nbsp;&nbsp;&nbsp;&nbsp;<span><font>作者：oneworld 转</font></span></td> 
  </tr> 
 </tbody> 
</table>

<table> 
 <tbody> 
  <tr> 
   <td style="WORD-WRAP: break-word" align="left"> <p style="TEXT-INDENT: 2em"><font></font><span></span></p> <p style="TEXT-INDENT: 2em">HTTP协议是目前互联网上最重要的协议，许多软件与服务都需要依赖HTTP协议。<br>虽然java.net这个package中包含了对HTTP的基本支持，但还有很多高级和复杂的功能无法实现，这不能不说是一个遗憾。<br>HttpClient作为Apache的开源项目项目之一，为基于HTTP协议的操作提供了强大的客户端执行支持，最新的版本为3.0RC3。<br>下面通过一个例子简要展示HttpClient的使用方法：<br><br>--------------------------------------------------------------------------------<br>import java.io.BufferedReader;<br>import java.io.IOException;<br></p> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> import java.io.InputStream;<br>import java.io.InputStreamReader;<br>iimport java.io.UnsupportedEncodingException;<br><br>import java.util.*;<br><br>import org.apache.commons.httpclient.Header;<br>import org.apache.commons.httpclient.HostConfiguration;<br>import org.apache.commons.httpclient.HttpClient;<br>import org.apache.commons.httpclient.HttpConnection;<br>import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;<br>import org.apache.commons.httpclient.NameValuePair;<br>import org.apache.commons.httpclient.methods.GetMethod;<br>import org.apache.commons.httpclient.methods.PostMethod;<br><br>/**<br>* @author steven<br>*/<br>public class HttpClientExample { <br><br>//获得ConnectionManager，设置相关参数<br>private static MultiThreadedHttpConnectionManager manager =<br>new MultiThreadedHttpConnectionManager();<br>private static int connectionTimeOut = 20000;<br>private static int socketTimeOut = 10000;<br>private static int maxConnectionPerHost = 5;<br>private static int maxTotalConnections = 40;<br><br>//标志初始化是否完成的flag<br>private static boolean initialed = false;<br><br>//初始化ConnectionManger的方法<br>public static void SetPara() { <br>manager.getParams().setConnectionTimeout(connectionTimeOut);<br>manager.getParams().setSoTimeout(socketTimeOut);<br>manager.getParams()<br>.setDefaultMaxConnectionsPerHost(maxConnectionPerHost);<br>manager.getParams().setMaxTotalConnections(maxTotalConnections);<br>initialed = true;<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> }<br><br>//通过get方法获取网页内容<br>public static String getGetResponseWithHttpClient(String url, String encode) { <br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> HttpClient client = new HttpClient(manager);<br><br>if (initialed) { <br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> HttpClientExample.SetPara();<br>}<br><br>GetMethod get = new GetMethod(url);<br>get.setFollowRedirects(true);<br><br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> String result = null;<br>StringBuffer resultBuffer = new StringBuffer();<br><br>try { <br><br>client.executeMethod(get);<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> <br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> //在目标页面情况未知的条件下，不推荐使用getResponseBodyAsString()方法<br>//String strGetResponseBody = post.getResponseBodyAsString();<br>BufferedReader in = new BufferedReader(<br>new InputStreamReader(<br>get.getResponseBodyAsStream(),<br>get.getResponseCharSet()));<br><br>String inputLine = null;<br><br>while ((inputLine = in.readLine()) != null) { <br>resultBuffer.append(inputLine);<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> resultBuffer.append("/n");<br>}<br><br>in.close();<br><br>result = resultBuffer.toString();<br><br>//iso-8859-1 is the default reading encode<br>result = HttpClientExample.ConverterStringCode(resultBuffer.toString(),<br>get.getResponseCharSet(),<br>encode);<br>} catch (Exception e) { <br>e.printStackTrace();<br><br>result = "";<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> } finally { <br>get.releaseConnection();<br><br>return result;<br>}<br>}<br><br>public static String getPostResponseWithHttpClient(String url,<br>String encode) { <br>HttpClient client = new HttpClient(manager);<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> <br>if (initialed) { <br>HttpClientExample.SetPara();<br>}<br><br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> PostMethod post = new PostMethod(url);<br>post.setFollowRedirects(false);<br><br>StringBuffer resultBuffer = new StringBuffer();<br><br>String result = null;<br><br>try { <br>client.executeMethod(post);<br><br>BufferedReader in = new BufferedReader(<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> new InputStreamReader(<br>post.getResponseBodyAsStream(),<br>post.getResponseCharSet()));<br>String inputLine = null;<br><br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> while ((inputLine = in.readLine()) != null) { <br>resultBuffer.append(inputLine);<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> resultBuffer.append("/n");<br>}<br><br>in.close();<br><br>//iso-8859-1 is the default reading encode<br>result = HttpClientExample.ConverterStringCode(resultBuffer.toString(),<br>post.getResponseCharSet(),<br>encode);<br>} catch (Exception e) { <br>e.printStackTrace();<br><br>result = "";<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> } finally { <br>post.releaseConnection();<br><br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> return result;<br>}<br>}<br><br>public static String getPostResponseWithHttpClient(String url,<br>String encode,<br>NameValuePair[] nameValuePair) { <br>HttpClient client = new HttpClient(manager);<br><br>if (initialed) { <br>HttpClientExample.SetPara();<br>}<br><br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> PostMethod post = new PostMethod(url);<br>post.setRequestBody(nameValuePair);<br>post.setFollowRedirects(false);<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> <br>String result = null;<br>StringBuffer resultBuffer = new StringBuffer();<br><br>try { <br>client.executeMethod(post);<br>BufferedReader in = new BufferedReader(<br>new InputStreamReader(<br>post.getResponseBodyAsStream(),<br>post.getResponseCharSet()));<br>String inputLine = null;<br><br>while ((inputLine = in.readLine()) != null) { <br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> resultBuffer.append(inputLine);<br>resultBuffer.append("/n");<br>}<br><br>in.close();<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> <br><br>//iso-8859-1 is the default reading encode<br>result = HttpClientExample.ConverterStringCode(resultBuffer.toString(),<br>post.getResponseCharSet(),<br>encode);<br>} catch (Exception e) { <br>e.printStackTrace();<br><br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> result = "";<br>} finally { <br>post.releaseConnection();<br><br>return result;<br>}<br>}<br><br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> private static String ConverterStringCode(String source, String srcEncode, String destEncode) { <br>if (src != null) { <br>try { <br>return new String(src.getBytes(srcEncode), destEncode);<br>} catch (UnsupportedEncodingException e) { <br>// TODO Auto-generated catch block<br>e.printStackTrace();<br>return "";<br>}<br>} else { <br>return "";<br> 
    <div style="VISIBILITY: hidden; OVERFLOW: hidden; WIDTH: 1px; COLOR: #f4fbff; HEIGHT: 1px">
      JAVA手机网[www.cnjm.net] 
    </div> }<br>}<br>}<br><br>--------------------------------------------------------------------------------<br>之后，就可以通过下面的代码获得目标网页：<br>String source = HttpClientExample.getGetResponseWithHttpClient("www.sina.com.cn", "GBK");<br><br>注意，在默认情况下，HttpClient的Request的Head中<br>User-Agent的值是Jakarta Commons-HttpClient 3.0RC1，如果需要改变它（例如，变为Mozilla/4.0），必须在调用之前运行如下语句：<br>System.getProperties().setProperty("httpclient.useragent", "Mozilla/4.0");<br></td> 
  </tr> 
 </tbody> 
</table>