spider简单的爬虫程序

不念不忘少年蓝@ 2022-08-04 12:44 190阅读 0赞

[http://blog.csdn.net/wyymaomi/archive/2008/12/03/3439066.aspx][http_blog.csdn.net_wyymaomi_archive_2008_12_03_3439066.aspx]

spider简单的爬虫程序

1、基础准备  
htmlparser  
首页：[http://sourceforge.net/projects/htmlparser/][http_sourceforge.net_projects_htmlparser]  
下载：[http://sourceforge.net/project/showfiles.php?group\_id=24399][http_sourceforge.net_project_showfiles.php_group_id_24399]  
文件：htmlparser1\_6\_20060610.zip  
<dependency>  
<groupId>org.htmlparser</groupId>  
<artifactId>htmlparser</artifactId>  
<version>1.6</version>  
</dependency>

cpdetector  
首页：[http://cpdetector.sourceforge.net/][http_cpdetector.sourceforge.net]  
下载：[http://sourceforge.net/project/showfiles.php?group\_id=114421][http_sourceforge.net_project_showfiles.php_group_id_114421]  
文件：cpdetector\_eclipse\_project\_1.0.7.zip

<dependency>  
<groupId>cpdetector</groupId>  
<artifactId>cpdetector</artifactId>  
<version>1.0.5</version>  
</dependency>

spindle  
首页：[http://www.bitmechanic.com/projects/spindle/][http_www.bitmechanic.com_projects_spindle] （但是已经无法访问）

2 修改spindle代码得到的spider  
简单的将URL打印出来了，解析的内容等等都没有处理

解析HTML的基类HtmlParserUtil.java

package com.sillycat.api.commons.utils.html;

import java.io.BufferedReader;  
import java.io.FileNotFoundException;  
import java.io.IOException;  
import java.io.InputStream;  
import java.io.InputStreamReader;  
import java.io.UnsupportedEncodingException;  
import java.net.MalformedURLException;  
import java.net.SocketException;  
import java.net.SocketTimeoutException;  
import java.net.URL;  
import java.net.UnknownHostException;  
import java.nio.charset.Charset;

import org.htmlparser.Parser;  
import org.htmlparser.util.NodeList;  
import org.htmlparser.util.ParserException;  
import org.htmlparser.visitors.HtmlPage;

import cpdetector.io.ASCIIDetector;  
import cpdetector.io.CodepageDetectorProxy;  
import cpdetector.io.JChardetFacade;  
import cpdetector.io.ParsingDetector;  
import cpdetector.io.UnicodeDetector;

public class HtmlParserUtil \{

/\* StringBuffer的缓冲区大小 \*/  
public static int TRANSFER\_SIZE = 4096;

/\* 当前平台的行分隔符 \*/  
public static String lineSep = System.getProperty("line.separator");

/\* 自动探测页面编码，避免中文乱码的出现 \*/  
public static String autoDetectCharset(URL url) \{

CodepageDetectorProxy detector = CodepageDetectorProxy.getInstance();  
   /\*\*  
   \* ParsingDetector可用于检查HTML、XML等文件或字符流的编码 构造方法中的参数用于指示是否显示探测过程的详细信息  
   \* 为false则不显示  
   \*/  
   detector.add(new ParsingDetector(false));  
   detector.add(JChardetFacade.getInstance());  
   detector.add(ASCIIDetector.getInstance());  
   detector.add(UnicodeDetector.getInstance());

Charset charset = null;  
   try \{  
    charset = detector.detectCodepage(url);  
   \} catch (MalformedURLException mue) \{  
    mue.printStackTrace();  
   \} catch (IOException ie) \{  
    ie.printStackTrace();  
   \}  
   if (charset == null)  
    charset = Charset.defaultCharset();  
   return charset.name();  
\}

/\* 按照指定编码解析标准的html页面，为建立索引做准备 \*/  
public static String\[\] parseHtml(String url, String charset) \{

String result\[\] = null;  
   String content = null;

try \{  
    URL source = new URL(url);  
    InputStream in = source.openStream();  
    BufferedReader reader = new BufferedReader(new InputStreamReader(  
      in, charset));  
    String line = new String();  
    StringBuffer temp = new StringBuffer(TRANSFER\_SIZE);  
    while ((line = reader.readLine()) != null) \{  
     temp.append(line);  
     temp.append(lineSep);  
    \}  
    reader.close();  
    in.close();  
    content = temp.toString();  
   \} catch (UnsupportedEncodingException uee) \{  
    uee.printStackTrace();  
   \} catch (MalformedURLException mue) \{  
    System.err.println("Invalid URL : " + url);  
   \} catch (UnknownHostException uhe) \{  
    System.err.println("UnknowHost : " + url);  
   \} catch (SocketException se) \{  
    System.err.println("Socket Error : " + se.getMessage() + " " + url);  
   \} catch (SocketTimeoutException ste) \{  
    System.err.println("Socket Connection Time Out : " + url);  
   \} catch (FileNotFoundException fnfe) \{  
    System.err.println("broken link "  
      + ((FileNotFoundException) fnfe.getCause()).getMessage()  
      + " ignored");  
   \} catch (IOException ie) \{  
    ie.printStackTrace();  
   \}

if (content != null) \{  
    Parser myParser = Parser.createParser(content, charset);  
    HtmlPage visitor = new HtmlPage(myParser);  
    try \{  
     myParser.visitAllNodesWith(visitor);  
     String body = null;  
     String title = "Untitled";  
     if (visitor.getBody() != null) \{  
      NodeList nodelist = visitor.getBody();  
      body = nodelist.asString().trim();  
     \}  
     if (visitor.getTitle() != null)\{  
      title = visitor.getTitle();  
     \}  
     result = new String\[\] \{ body, title \};  
    \} catch (ParserException pe) \{  
     pe.printStackTrace();  
    \}  
   \}  
   return result;  
\}  
\}

多线程爬虫类    HtmlCaptureRunner.java

package com.sillycat.api.thread.runner;

import java.io.FileNotFoundException;  
import java.io.IOException;  
import java.net.HttpURLConnection;  
import java.net.MalformedURLException;  
import java.net.SocketException;  
import java.net.SocketTimeoutException;  
import java.net.URL;  
import java.net.UnknownHostException;  
import java.util.ArrayList;  
import java.util.HashSet;

import org.apache.commons.logging.Log;  
import org.apache.commons.logging.LogFactory;  
import org.htmlparser.Parser;  
import org.htmlparser.PrototypicalNodeFactory;  
import org.htmlparser.filters.AndFilter;  
import org.htmlparser.filters.HasAttributeFilter;  
import org.htmlparser.filters.NodeClassFilter;  
import org.htmlparser.tags.BaseHrefTag;  
import org.htmlparser.tags.FrameTag;  
import org.htmlparser.tags.LinkTag;  
import org.htmlparser.tags.MetaTag;  
import org.htmlparser.util.EncodingChangeException;  
import org.htmlparser.util.NodeIterator;  
import org.htmlparser.util.NodeList;  
import org.htmlparser.util.ParserException;

import com.sillycat.api.commons.utils.StringUtil;  
import com.sillycat.api.commons.utils.html.HtmlParserUtil;

public class HtmlCaptureRunner implements Runnable \{

public Log logger = LogFactory.getLog(getClass());

/\* 基准(初始)URL \*/  
protected String baseURL = null;

private String contentPath = null;

/\*\*  
\* 待解析的URL地址集合，所有新检测到的链接均存放于此； 解析时按照先入先出（First-In First-Out）法则线性取出  
\*/  
protected ArrayList URLs = new ArrayList();

/\* 已存储的URL地址集合，避免链接的重复抓取 \*/  
protected HashSet indexedURLs = new HashSet();

protected Parser parser = new Parser();;

/\* 程序运行线程数，默认2个线程 \*/  
protected int threads = 2;

/\* 解析页面时的字符编码 \*/  
protected String charset;

/\* 基准端口 \*/  
protected int basePort;

/\* 基准主机 \*/  
protected String baseHost;

/\* 是否存储,默认true \*/  
protected boolean justDatabase = true;

/\* 检测索引中是否存在当前URL信息,避免重复抓取 \*/  
protected boolean isRepeatedCheck = false;

public HtmlCaptureRunner() \{  
   PrototypicalNodeFactory factory = new PrototypicalNodeFactory();  
   factory.registerTag(new LocalLinkTag());  
   factory.registerTag(new LocalFrameTag());  
   factory.registerTag(new LocalBaseHrefTag());  
   parser.setNodeFactory(factory);  
\}

public void capture() \{  
   URLs.clear();  
   URLs.add(getBaseURL());

int responseCode = 0;  
   String contentType = "";

try \{  
    HttpURLConnection uc = (HttpURLConnection) new URL(baseURL)  
      .openConnection();  
    responseCode = uc.getResponseCode();  
    contentType = uc.getContentType();  
   \} catch (MalformedURLException mue) \{  
    logger.error("Invalid URL : " + getBaseURL());  
   \} catch (UnknownHostException uhe) \{  
    logger.error("UnknowHost : " + getBaseURL());  
   \} catch (SocketException se) \{  
    logger.error("Socket Error : " + se.getMessage() + " "  
      + getBaseURL());  
   \} catch (IOException ie) \{  
    logger.error("IOException : " + ie);  
   \}

if (responseCode == HttpURLConnection.HTTP\_OK  
     && contentType.startsWith("text/html")) \{  
    try \{  
     charset = HtmlParserUtil.autoDetectCharset(new URL(baseURL));

basePort = new URL(baseURL).getPort();  
     baseHost = new URL(baseURL).getHost();  
     if (charset.equals("windows-1252"))  
      charset = "GBK";

long start = System.currentTimeMillis();  
     ArrayList threadList = new ArrayList();  
     for (int i = 0; i < threads; i++) \{  
      Thread t = new Thread(this, "Spider Thread \#" + (i + 1));  
      t.start();  
      threadList.add(t);  
     \}  
     while (threadList.size() > 0) \{  
      Thread child = (Thread) threadList.remove(0);  
      try \{  
       child.join();  
      \} catch (InterruptedException ie) \{  
       logger.error("InterruptedException : " + ie);  
      \}  
     \}  
     // for (int i = 0; i < threads; i++) \{  
     // threadPool.getThreadPoolExcutor().execute(new  
     // Thread(this,"Spider Thread \#" + (i + 1)));  
     // \}  
     long elapsed = System.currentTimeMillis() - start;  
     logger.info("Finished in " + (elapsed / 1000) + " seconds");  
     logger.info("The Count of the Links Captured is "  
       + indexedURLs.size());  
    \} catch (MalformedURLException e) \{  
     e.printStackTrace();  
    \}  
   \}  
\}

public void run() \{  
   String url;  
   while ((url = dequeueURL()) != null) \{  
    if (justDatabase) \{  
     process(url);  
    \}  
   \}  
   threads--;  
\}

/\*\*  
\* 处理单独的URL地址，解析页面并加入到lucene索引中；通过自动探测页面编码保证抓取工作的顺利执行  
\*/  
protected void process(String url) \{

String result\[\];  
   String content = null;  
   String title = null;

result = HtmlParserUtil.parseHtml(url, charset);  
   content = result\[0\];  
   title = result\[1\];

if (content != null && content.trim().length() > 0) \{  
    // content  
    System.out.println(url);  
    // title  
    // DateTools.timeToString(System.currentTimeMillis()  
   \}  
\}

/\* 从URL队列mPages里取出单个的URL \*/  
public synchronized String dequeueURL() \{  
   while (true)  
    if (URLs.size() > 0) \{  
     String url = (String) URLs.remove(0);  
     indexedURLs.add(url);  
     if (isToBeCaptured(url)) \{  
      NodeList list;  
      try \{  
       int bookmark = URLs.size();  
       /\* 获取页面所有节点 \*/  
       parser.setURL(url);  
       try \{  
        list = new NodeList();  
        for (NodeIterator e = parser.elements(); e  
          .hasMoreNodes();)  
         list.add(e.nextNode());  
       \} catch (EncodingChangeException ece) \{  
        /\* 解码出错的异常处理 \*/  
        parser.reset();  
        list = new NodeList();  
        for (NodeIterator e = parser.elements(); e  
          .hasMoreNodes();)  
         list.add(e.nextNode());  
       \}  
       /\*\*  
       \* 依据 [http://www.robotstxt.org/wc/meta-user.html][http_www.robotstxt.org_wc_meta-user.html] 处理  
       \* Robots <META> tag  
       \*/  
       NodeList robots = list  
         .extractAllNodesThatMatch(  
           new AndFilter(new NodeClassFilter(  
             MetaTag.class),  
             new HasAttributeFilter("name",  
               "robots")), true);  
       if (0 != robots.size()) \{  
        MetaTag robot = (MetaTag) robots.elementAt(0);  
        String content = robot.getAttribute("content")  
          .toLowerCase();  
        if ((-1 != content.indexOf("none"))  
          || (-1 != content.indexOf("nofollow")))  
         for (int i = bookmark; i < URLs.size(); i++)  
          URLs.remove(i);  
       \}  
      \} catch (ParserException pe) \{  
       logger.error("ParserException : " + pe);  
      \}  
      return url;  
     \}  
    \} else \{  
     threads--;  
     if (threads > 0) \{  
      try \{  
       wait();  
       threads++;  
      \} catch (InterruptedException ie) \{  
       logger.error("InterruptedException : " + ie);  
      \}  
     \} else \{  
      notifyAll();  
      return null;  
     \}  
    \}  
\}

private boolean isHTML(String url) \{  
   if (!url.endsWith(".html")) \{  
    return false;  
   \}  
   if (StringUtil.isNotBlank(contentPath)) \{  
    if (!url.startsWith(baseURL + "/" + contentPath)) \{  
     return false;  
    \}  
   \}  
   return true;  
\}

/\*\*  
\* 判断提取到的链接是否符合解析条件；标准为Port及Host与基准URL相同且类型为text/html或text/plain  
\*/  
public boolean isToBeCaptured(String url) \{

boolean flag = false;

HttpURLConnection uc = null;  
   int responseCode = 0;  
   String contentType = "";  
   String host = "";  
   int port = 0;

try \{  
    URL source = new URL(url);  
    String protocol = source.getProtocol();  
    if (protocol != null && protocol.equals("http")) \{  
     host = source.getHost();  
     port = source.getPort();  
     uc = (HttpURLConnection) source.openConnection();  
     uc.setConnectTimeout(8000);  
     responseCode = uc.getResponseCode();  
     contentType = uc.getContentType();  
    \}  
   \} catch (MalformedURLException mue) \{  
    logger.error("Invalid URL : " + url);  
   \} catch (UnknownHostException uhe) \{  
    logger.error("UnknowHost : " + url);  
   \} catch (SocketException se) \{  
    logger.error("Socket Error : " + se.getMessage() + " " + url);  
   \} catch (SocketTimeoutException ste) \{  
    logger.error("Socket Connection Time Out : " + url);  
   \} catch (FileNotFoundException fnfe) \{  
    logger.error("broken link " + url + " ignored");  
   \} catch (IOException ie) \{  
    logger.error("IOException : " + ie);  
   \}  
   if (port == basePort  
     && responseCode == HttpURLConnection.HTTP\_OK  
     && host.equals(baseHost)  
     && (contentType.startsWith("text/html") || contentType  
       .startsWith("text/plain")))  
    flag = true;  
   return flag;  
\}

class LocalLinkTag extends LinkTag \{  
   public void doSemanticAction() \{  
    String link = getLink();  
    if (link.endsWith("/"))  
     link = link.substring(0, link.length() - 1);  
    int pos = link.indexOf("\#");  
    if (pos != -1)  
     link = link.substring(0, pos);  
    /\* 将链接加入到处理队列中 \*/  
    if (!(indexedURLs.contains(link) || URLs.contains(link))) \{  
     if (isHTML(link)) \{  
      URLs.add(link);  
     \}  
    \}  
    setLink(link);  
   \}  
\}

/\*\*  
\* Frame tag that rewrites the SRC URLs. The SRC URLs are mapped to local  
\* targets if they match the source.  
\*/  
class LocalFrameTag extends FrameTag \{  
   public void doSemanticAction() \{  
    String link = getFrameLocation();  
    if (link.endsWith("/"))  
     link = link.substring(0, link.length() - 1);  
    int pos = link.indexOf("\#");  
    if (pos != -1)  
     link = link.substring(0, pos);  
    /\* 将链接加入到处理队列中 \*/  
    if (!(indexedURLs.contains(link) || URLs.contains(link))) \{  
     if (isHTML(link)) \{  
      URLs.add(link);  
     \}  
    \}  
    setFrameLocation(link);  
   \}  
\}

/\*\*  
\* Base tag that doesn't show. The toHtml() method is overridden to return  
\* an empty string, effectively shutting off the base reference.  
\*/  
class LocalBaseHrefTag extends BaseHrefTag \{  
   public String toHtml() \{  
    return ("");  
   \}  
\}

public String getBaseURL() \{  
   return baseURL;  
\}

public void setBaseURL(String baseURL) \{  
   this.baseURL = baseURL;  
\}

public int getThreads() \{  
   return threads;  
\}

public void setThreads(int threads) \{  
   this.threads = threads;  
\}

public String getCharset() \{  
   return charset;  
\}

public void setCharset(String charset) \{  
   this.charset = charset;  
\}

public int getBasePort() \{  
   return basePort;  
\}

public void setBasePort(int basePort) \{  
   this.basePort = basePort;  
\}

public String getBaseHost() \{  
   return baseHost;  
\}

public void setBaseHost(String baseHost) \{  
   this.baseHost = baseHost;  
\}

public boolean isJustDatabase() \{  
   return justDatabase;  
\}

public void setJustDatabase(boolean justDatabase) \{  
   this.justDatabase = justDatabase;  
\}

public String getContentPath() \{  
   return contentPath;  
\}

public void setContentPath(String contentPath) \{  
   this.contentPath = contentPath;  
\}

spring上的配置文件applicationContext-bean.xml:  
<bean id="productCapture"  
   class="com.sillycat.api.thread.runner.HtmlCaptureRunner" >  
   <property name="contentPath" value="$\{product.contentPath\}" />  
   <property name="basePort" value="$\{product.base.port\}" />  
   <property name="baseURL" value="$\{product.base.url\}" />  
   <property name="charset" value="$\{product.base.code\}" />  
   <property name="threads" value="$\{product.base.threads\}"/>  
</bean>

easySearch.properties配置文件：  
\#==========================================  
\# spider configration  
\#=========================================  
product.contentPath=product  
product.base.port=80  
product.base.url=http://www.safedv.com  
product.base.code=UTF-8  
product.base.threads=3

message.contentPath=message  
message.base.port=80  
message.base.url=http://www.safedv.com  
message.base.code=UTF-8  
message.base.threads=3

单元测试类HtmlRunnerTest.java文件：

package com.sillycat.api.thread;

import com.sillycat.api.commons.base.BaseManagerTest;  
import com.sillycat.api.thread.runner.HtmlCaptureRunner;

public class HtmlRunnerTest extends BaseManagerTest \{

private HtmlCaptureRunner productCapture;

private HtmlCaptureRunner messageCapture;

protected void setUp() throws Exception \{  
   super.setUp();  
   productCapture = (HtmlCaptureRunner) appContext.getBean("productCapture");  
   messageCapture = (HtmlCaptureRunner) appContext.getBean("messageCapture");  
\}

protected void tearDown() throws Exception \{  
   super.tearDown();  
\}

public void testDumy() \{  
   assertTrue(true);  
\}

public void ntestProductCapture() \{  
   productCapture.capture();  
\}

public void testMessageCapture()\{  
   messageCapture.capture();  
\}  
\}

本文来自CSDN博客，转载请标明出处：[http://blog.csdn.net/wyymaomi/archive/2008/12/03/3439066.aspx][http_blog.csdn.net_wyymaomi_archive_2008_12_03_3439066.aspx]

一个简单的java网络爬虫(spider)

[http://blog.csdn.net/wyymaomi/archive/2008/12/03/3439016.aspx][http_blog.csdn.net_wyymaomi_archive_2008_12_03_3439016.aspx]

一个简单的java网络爬虫,由于时间原因,没有进一步解释.

需要的htmlparser.jar包到官方网上去下.

\---------------------------------------------Spider.java-----------------------------------------------------------------

import java.io.BufferedReader;  
import java.io.InputStreamReader;  
import java.net.URL;  
import java.net.URLConnection;  
import java.util.ArrayList;  
import java.util.HashMap;  
import java.util.Iterator;  
import java.util.List;  
import org.htmlparser.RemarkNode;  
import org.htmlparser.StringNode;  
import org.htmlparser.Node;  
import org.htmlparser.tags.\*;  
import org.htmlparser.Parser;  
import org.htmlparser.filters.StringFilter;  
import org.htmlparser.util.NodeIterator;  
import org.htmlparser.util.NodeList;  
import org.htmlparser.util.ParserException;  
import java.util.Queue;  
import java.util.LinkedList;

public class Spider implements Runnable \{

boolean search\_key\_words = false;

int count = 0;

int limitsite = 10;

int countsite = 1;

String keyword = "中国";//搜索关键字

Parser parser = new Parser();

// List linklist = new ArrayList();  
String startsite = "";//搜索的其实站点

SearchResultBean srb;//保存搜索结果

List resultlist = new ArrayList();//搜索到关键字链接列表

List searchedsite = new ArrayList();//已经被搜索站点列表

Queue linklist = new LinkedList();//需解析的链接列表

HashMap<String, ArrayList<String>> disallowListCache = new HashMap<String, ArrayList<String>>();

public Spider(String keyword, String startsite) \{  
   this.keyword = keyword;  
   this.startsite = startsite;  
   linklist.add(startsite);  
   srb = new SearchResultBean();  
\}

public void run() \{  
   // TODO Auto-generated method stub  
   search(linklist);

public void search(Queue queue) \{  
   String url = "";  
     while(!queue.isEmpty())\{  
    url = queue.peek().toString();//查找列队  
    try \{  
     if (!isSearched(searchedsite, url)) \{  
      if (isRobotAllowed(new URL(url)))//检查该链接是否被允许搜索  
       processHtml(url);  
      else  
       System.out.println("this page is disallowed to search");  
     \}  
    \} catch (Exception ex) \{

\}  
    queue.remove();  
    
     \}  
      
\}  
/\*\*  
\* 解析HTML  
\* @param url   
\* @throws ParserException  
\* @throws Exception  
\*/  
public void processHtml(String url) throws ParserException, Exception \{  
   searchedsite.add(url);  
   count = 0;  
   System.out.println("searching ... :" + url);  
   parser.setURL(url);  
   parser.setEncoding("GBK");  
   URLConnection uc = parser.getConnection();  
   uc.connect();  
   //uc.getLastModified();  
   NodeIterator nit = parser.elements();  
    
   while (nit.hasMoreNodes()) \{  
    Node node = nit.nextNode();  
    parserNode(node);  
   \}  
   srb.setKeywords(keyword);  
   srb.setUrl(url);  
   srb.setCount\_key\_words(count);  
   resultlist.add(srb);  
   System.out.println("count keywords is :" + count);  
   System.out.println("----------------------------------------------");  
\}  
/\*\*  
\* 处理HTML标签  
\* @param tag  
\* @throws Exception  
\*/  
public void dealTag(Tag tag) throws Exception \{  
   NodeList list = tag.getChildren();  
   if (list != null) \{  
    NodeIterator it = list.elements();  
    while (it.hasMoreNodes()) \{  
     Node node = it.nextNode();  
     parserNode(node);  
    \}  
   \}  
\}  
/\*\*  
\* 处理HTML标签结点  
\* @param node  
\* @throws Exception  
\*/  
    public void parserNode(Node node) throws Exception\{  
    if (node instanceof StringNode) \{//判断是否是文本结点  
    StringNode sNode = (StringNode) node;  
    StringFilter sf = new StringFilter(keyword,false);  
    search\_key\_words = sf.accept(sNode);  
    if (search\_key\_words) \{  
     count++;  
    \}  
    // System.out.println("text is :"+sNode.getText().trim());  
   \} else if (node instanceof Tag) \{//判断是否是标签库结点  
    Tag atag = (Tag) node;  
    if (atag instanceof TitleTag) \{//判断是否是标TITLE结点  
     srb.setTitle(atag.getText());  
    \}  
    if (atag instanceof LinkTag) \{//判断是否是标LINK结点  
     LinkTag linkatag = (LinkTag) atag;  
     checkLink(linkatag.getLink(), linklist);  
     // System.out.println("-----------------this is link --------------");  
    \}  
    dealTag(atag);  
   \} else if (node instanceof RemarkNode) \{//判断是否是注释  
    // System.out.println("this is remark");  
   \}  
    \}  
    /\*  
     \* 检查链接是否需要加入列队  
     \*/  
public void checkLink(String link, Queue queue) \{  
   if (link != null && !link.equals("") && link.indexOf("\#") == -1) \{  
    if (!link.startsWith("http://") && !link.startsWith("ftp://")  
      && !link.startsWith("www.")) \{  
     link = "file:///" + link;  
    \} else if (link.startsWith("www.")) \{  
     link = "http://" + link;  
    \}  
    if (queue.isEmpty())  
     queue.add(link);  
    else \{  
     String link\_end\_=link.endsWith("/")?link.substring(0,link.lastIndexOf("/")):(link+"/");  
     if (!queue.contains(link)&&!queue .contains(link\_end\_)) \{  
      queue.add(link);  
     \}  
    \}  
   \}  
\}  
/\*\*  
\* 检查该链接是否已经被扫描  
\* @param list  
\* @param url  
\* @return  
\*/  
public boolean isSearched(List list, String url) \{  
   String url\_end\_ = "";  
   if (url.endsWith("/")) \{  
    url\_end\_ = url.substring(0, url.lastIndexOf("/"));  
   \} else \{  
    url\_end\_ = url + "/";  
   \}  
   if (list.size() > 0) \{  
    if (list.indexOf(url) != -1 || list.indexOf(url\_end\_) != -1) \{  
     return true;  
    \}  
   \}  
   return false;  
\}  
/\*\*  
\* 检查URL是否被允许搜索  
\* @param urlToCheck  
\* @return  
\*/  
private boolean isRobotAllowed(URL urlToCheck) \{  
   String host = urlToCheck.getHost().toLowerCase();// 获取给出RUL的主机  
   // System.out.println("主机="+host);

// 获取主机不允许搜索的URL缓存  
   ArrayList<String> disallowList = disallowListCache.get(host);

// 如果还没有缓存,下载并缓存。  
   if (disallowList == null) \{  
    disallowList = new ArrayList<String>();  
    try \{  
     URL robotsFileUrl = new URL("http://" + host + "/robots.txt");  
     BufferedReader reader = new BufferedReader(  
       new InputStreamReader(robotsFileUrl.openStream()));

// 读robot文件，创建不允许访问的路径列表。  
     String line;  
     while ((line = reader.readLine()) != null) \{  
      if (line.indexOf("Disallow:") == 0) \{// 是否包含"Disallow:"  
       String disallowPath = line.substring("Disallow:"  
         .length());// 获取不允许访问路径

// 检查是否有注释。  
       int commentIndex = disallowPath.indexOf("\#");  
       if (commentIndex != -1) \{  
        disallowPath = disallowPath.substring(0,  
          commentIndex);// 去掉注释  
       \}

disallowPath = disallowPath.trim();  
       disallowList.add(disallowPath);  
      \}  
     \}  
     for (Iterator it = disallowList.iterator(); it.hasNext();) \{  
      System.out.println("Disallow is :" + it.next());  
     \}  
     // 缓存此主机不允许访问的路径。  
     disallowListCache.put(host, disallowList);  
    \} catch (Exception e) \{  
     return true; // web站点根目录下没有robots.txt文件,返回真  
    \}  
   \}

String file = urlToCheck.getFile();  
   // System.out.println("文件getFile()="+file);  
   for (int i = 0; i < disallowList.size(); i++) \{  
    String disallow = disallowList.get(i);  
    if (file.startsWith(disallow)) \{  
     return false;  
    \}  
   \}

return true;  
\}

public static void main(String\[\] args) \{

Spider ph = new Spider("英超", "[http://www.microsoft.com][http_www.microsoft.com]");  
   try \{  
    // ph.processHtml();  
    Thread search = new Thread(ph);  
    search.start();//启动线程  
   \} catch (Exception ex) \{

\}  
\}

\--------------------------------------SearchResultBean.java---------------------------------------------------------

public class SearchResultBean \{  
   String url = "";  
   String title = "";  
   String keywords = "";  
   int count\_key\_words = 0;  
public int getCount\_key\_words() \{  
return count\_key\_words;  
\}  
public void setCount\_key\_words(int count\_key\_words) \{  
this.count\_key\_words = count\_key\_words;  
\}  
public String getKeywords() \{  
return keywords;  
\}  
public void setKeywords(String keywords) \{  
this.keywords = keywords;  
\}  
public String getTitle() \{  
return title;  
\}  
public void setTitle(String title) \{  
this.title = title;  
\}  
public String getUrl() \{  
return url;  
\}  
public void setUrl(String url) \{  
this.url = url;  
\}  
\}

本文来自CSDN博客，转载请标明出处：[http://blog.csdn.net/wyymaomi/archive/2008/12/03/3439016.aspx][http_blog.csdn.net_wyymaomi_archive_2008_12_03_3439016.aspx]

[http_blog.csdn.net_wyymaomi_archive_2008_12_03_3439066.aspx]: http://blog.csdn.net/wyymaomi/archive/2008/12/03/3439066.aspx
[http_sourceforge.net_projects_htmlparser]: http://sourceforge.net/projects/htmlparser/
[http_sourceforge.net_project_showfiles.php_group_id_24399]: http://sourceforge.net/project/showfiles.php?group_id=24399
[http_cpdetector.sourceforge.net]: http://cpdetector.sourceforge.net/
[http_sourceforge.net_project_showfiles.php_group_id_114421]: http://sourceforge.net/project/showfiles.php?group_id=114421
[http_www.bitmechanic.com_projects_spindle]: http://www.bitmechanic.com/projects/spindle/
[http_www.robotstxt.org_wc_meta-user.html]: http://www.robotstxt.org/wc/meta-user.html
[http_blog.csdn.net_wyymaomi_archive_2008_12_03_3439016.aspx]: http://blog.csdn.net/wyymaomi/archive/2008/12/03/3439016.aspx
[http_www.microsoft.com]: http://www.microsoft.com