网络爬虫基础！

深碍√TFBOYSˉ_ 2022-08-03 15:43 213阅读 0赞

网络爬虫最基础的就是下载文件或页面，然后利用正则表达式提取需要的数据！

假如有如下一个mail.txt文件:需要从里面提取有用的邮箱地址，该怎么做呢？

skdajflksadjflk> abc0@sina.com<djfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajfl> abc1@sina.com<ksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajfl> abc2@sina.com<sadjflkdjfsa;lkdfjks> abc3@sina.com<a;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjflkdjfsa;lkv> ab4c@sina.com<dfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajf> abc5@sina.com<ksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjf> abc6@sina.com<lkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjflkdjfsa;lkd> abc7@sina.com<fjksa;ljfs;alfj> abc8@sina.com<sa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflks> abc11@sina.com<adjflkdjfsa;> abc9@sina.com<lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjf> abc12@sina.com<lkdjfsa;lk> abc55@sina.com<dfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
    skdajflksadjflk> abc123@sina.com<djfsa;lkdfjksa;ljfs;alfjsa;lkjfs> abc1234@sina.com<a;lkfjska;ljfsafsd
    sdfsakfjask;lfjsak;lfj qqq@sina.com aslk;fjsafsafsafs

我们只需要下载这个页面，然后解析其中的邮箱地址即可！

public class TestRegex {
    	//获取指定文档中的邮件地址
    	public static void getMails() throws IOException {
    		BufferedReader bufr = new BufferedReader(new FileReader("D:\\mail.txt"));
    		String line = null;
    		String mailReg = "\\w+@\\w+(\\.\\w)+";
    		Pattern p = Pattern.compile(mailReg);
    		
    		while((line = bufr.readLine()) != null) {
    			Matcher m = p.matcher(line);
    			while(m.find()) {
    				System.out.println(m.group());
    			}
    		}
    		
    	}
    	
    	public static void main(String[] args) throws IOException {
    		getMails();
    	}
    }

好了，一个简单的网络爬虫就是这样的了！

如果要下载web页面呢？可以使用urlconnection进行下载：

public class TestRegex2 {
    	public static void getHtml() throws IOException {
    		URL url = new URL("http://s.taobao.com/search?q=%E8%A1%AC%E8%A1%AB&commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&spm=1.7274553.1997520841.1&initiative_id=tbindexz_20150603");
    		URLConnection conn = url.openConnection();
    		BufferedReader bufferedReader = new BufferedReader(
    				new InputStreamReader(conn.getInputStream()));
    		String line = null;
    //		String mailReg = "g_page_config = ";
    //		Pattern p = Pattern.compile(mailReg);
    		
    		while((line = bufferedReader.readLine()) != null) {
    //			Matcher m = p.matcher(line);
    //			while(m.find())
    //			{
    //				System.out.println(m.group());
    //			}
    			System.out.println(line);
    		}
    	}
    	
    	public static void main(String[] args) throws IOException {
    		getHtml();
    	}
    }