花瓣网图片采集

Myth丶恋晨 2022-01-10 09:27 211阅读 0赞

这段时间一直研究新闻采集，正好花瓣网刚刚兴起，突然萌发搞个程序下载花瓣网上图片的想法，程序很简单，希望大家指点，以及完善，共同提高。目前存在的问题是花瓣网采用分屏ajax处理，顾该采集只能采集第一页首页所包含的所有图片，正在研究当中，希望大家有好的想法或者处理方法，也多多指点和告知。

1.  /\*\*
2.  \* 内容摘要：花瓣网采集类
3.  \* 流程说明：
4.  \* @author haozi  
    
5.  \* @return
6.  \*/
7.  package test;
8.  
9.  import org.htmlparser.Node;
10. import org.htmlparser.tags.ImageTag;
11. import org.htmlparser.tags.LinkTag;
12. 
13. import util.CrawNewsTools;
14. 
15. public class CrawPicForHuaBan \{
16. private static String baseUrl = "http://huaban.com";
17. //从首页采集包含图片的URL列表
18. public void crawImageList(String indexUrl, String encode) \{
19. // 采集源码
20. String page = CrawNewsTools.getPage(indexUrl, encode);
21. System.out.println("\#\#\#采集到首页的内容:" + page);
22. String templateNewsUrl = "/pins/(.\*)";
23. Node\[\] nodes = CrawNewsTools.getTagList(page, encode, LinkTag.class);
24. for (int i = 0; i < nodes.length; i++) \{
25. String link = ((LinkTag) nodes\[i\]).getLink();
26. boolean isRegex = CrawNewsTools.isRegex(link, templateNewsUrl);
27. if (isRegex) \{
28. String completeLink = baseUrl \+ link;
29. System.out.println("\#\#\#符合规则的URL：" + completeLink);
30. crawImage(completeLink, encode);
31. \}
32. \}
33. 
34. \}
35. //采集图片
36. public void crawImage(String url, String encode) \{
37. // 采集源码
38. String page = CrawNewsTools.getPage(url, encode);
39. String formatPage = CrawNewsTools.fromatBodyTag(page);
40. String bodyPage = CrawNewsTools.getBody(formatPage);
41. String body = CrawNewsTools.formatHtml(bodyPage);
42. System.out.println("\#\#\#格式后body源码:\\r\\n" + body);
43. Node\[\] nodes = CrawNewsTools.getTagList(body, encode, ImageTag.class);
44. for (int i = 0; i < nodes.length; i++) \{
45. String picUrl = ((ImageTag) nodes\[i\]).getImageURL();
46. String patternUrl = "http://img.hb.aicdn.com/(.\*)";
47. boolean isRegex = CrawNewsTools.isRegex(picUrl, patternUrl);
48. if (isRegex) \{
49. System.out.println("图片URL：" + picUrl);
50. \}
51. \}
52. \}
53. 
54. public static void main(String\[\] args) \{
55. String url = "http://huaban.com/all/beauty/";
56. String encode = "UTF-8";
57. new CrawPicForHuaBan().crawImageList(url, encode);
58. \}
59. 
60. \}
61. 
62. 
63. /\*\*
64. \* 内容摘要：新闻采集工具类
65. \* 流程说明：
66. \* @author haozi
67. \* @return
68. \*/
69. package util;
70. 
71. import java.util.regex.Matcher;
72. import java.util.regex.Pattern;
73. 
74. import org.apache.http.HttpEntity;
75. import org.apache.http.HttpResponse;
76. import org.apache.http.client.HttpClient;
77. import org.apache.http.client.methods.HttpGet;
78. import org.apache.http.impl.client.DefaultHttpClient;
79. import org.apache.http.util.EntityUtils;
80. import org.htmlparser.Node;
81. import org.htmlparser.Parser;
82. import org.htmlparser.tags.ScriptTag;
83. import org.htmlparser.tags.StyleTag;
84. import org.htmlparser.visitors.ObjectFindingVisitor;
85. import org.htmlparser.visitors.TextExtractingVisitor;
86. 
87. public class CrawNewsTools \{
88. 
89. // 使用HttpClient组件读取指定URL的页面HTML源码
90. public static String getPage(String url, String encode) \{
91. String page = "";
92. HttpClient httpClient = null;
93. try \{
94. httpClient = new DefaultHttpClient();
95. // 创建httpget
96. HttpGet httpget = new HttpGet(url);
97. System.out.println("请求URI路径:" + httpget.getURI());
98. // 执行get请求
99. HttpResponse response = httpClient.execute(httpget);
100. // 获得响应实体
101. HttpEntity httpEntity = response.getEntity();
102. String charset = EntityUtils.getContentCharSet(httpEntity);
103. System.out.println("\#\#\#当前页面编码:" + charset);
104. // 获取内容时，指定编码
105. if (encode != null && !encode.trim().equals("")) \{
106. System.out.println("\#\#\#采用指定编码:" + encode);
107. \} else if (charset != null) \{
108. System.out.println("\#\#\#采用网页自身编码:" + charset);
109. encode = charset;
110. \} else \{
111. System.out.println("\#\#\#采用默认UTF-8编码:UTF-8");
112. encode = "UTF-8";
113. \}
114. page = EntityUtils.toString(httpEntity, encode);
115. page = removeCssTag(page, encode);
116. page = removeJsTag(page, encode);
117. page = getBody(page);
118. page = formatHtml(page);
119. \} catch (Exception ex) \{
120. ex.printStackTrace();
121. \} finally \{
122. httpClient.getConnectionManager().shutdown();
123. \}
124. return page;
125. \}
126. 
127. // 格式化body标签
128. public static String fromatBodyTag(String htmlCode) \{
129. String result = htmlCode;
130. if (htmlCode != null) \{
131. while (result.indexOf("<BODY") != -1) \{
132. result.replaceAll("<BODY", "<body");
133. \}
134. while (result.indexOf("<Body") != -1) \{
135. resultresult = result.replaceAll("<Body", "<body");
136. \}
137. while (result.indexOf("</BODY") != -1) \{
138. resultresult = result.replaceAll("</BODY", "</body");
139. \}
140. while (result.indexOf("</Body") != -1) \{
141. resultresult = result.replaceAll("</Body", "</body");
142. \}
143. \}
144. return result;
145. \}
146. 
147. // 使用正则表达式提取body体内容
148. public static String getBody(String htmlCode) \{
149. if (htmlCode == null) \{
150. return null;
151. \}
152. Pattern pattern = Pattern.compile("<body(.\*)>(.\*)</body>",
153. Pattern.MULTILINE | Pattern.DOTALL);
154. Matcher matcher = pattern.matcher(htmlCode);
155. if (matcher.find()) \{
156. return matcher.group();
157. \} else \{
158. return null;
159. \}
160. \}
161. 
162. // 过滤css标签
163. public static String removeCssTag(String htmlCode, String encode) \{
164. String htmlEndCode = htmlCode;
165. try \{
166. Parser parser = Parser.createParser(htmlCode, encode);
167. ObjectFindingVisitor visitor = new ObjectFindingVisitor(
168. StyleTag.class);
169. parser.visitAllNodesWith(visitor);
170. Node\[\] nodes = visitor.getTags();
171. for (int i = 0; i < nodes.length; i++) \{
172. // System.out.println(nodes\[i\].toHtml());
173. htmlEndCodehtmlEndCode = htmlEndCode.replace(nodes\[i\].toHtml(), "");
174. \}
175. // System.out.println("\#\#\#去除css标签后:" + htmlEndCode);
176. \} catch (Exception e) \{
177. e.printStackTrace();
178. \}
179. return htmlEndCode;
180. \}
181. 
182. // 过滤js标签
183. public static String removeJsTag(String htmlCode, String encode) \{
184. String htmlEndCode = htmlCode;
185. try \{
186. Parser parser = Parser.createParser(htmlCode, encode);
187. ObjectFindingVisitor visitor = new ObjectFindingVisitor(
188. ScriptTag.class);
189. parser.visitAllNodesWith(visitor);
190. Node\[\] nodes = visitor.getTags();
191. for (int i = 0; i < nodes.length; i++) \{
192. // System.out.println(nodes\[i\].toHtml());
193. htmlEndCodehtmlEndCode = htmlEndCode.replace(nodes\[i\].toHtml(), "");
194. \}
195. // System.out.println("\#\#\#去除js标签后:" + htmlEndCode);
196. \} catch (Exception e) \{
197. e.printStackTrace();
198. \}
199. return htmlEndCode;
200. \}
201. 
202. // 格式化指定的HTML源码
203. public static String formatHtml(String htmlcode) \{
204. String result = htmlcode;
205. if (htmlcode != null && htmlcode.trim().length() > 0) \{
206. // 去除回车符
207. while (result.indexOf("\\r") != -1) \{
208. resultresult = result.replaceAll("\\r", "");
209. \}
210. // 去除换行符
211. while (result.indexOf("\\n") != -1) \{
212. resultresult = result.replaceAll("\\n", "");
213. \}
214. // 去除制表符
215. while (result.indexOf("\\t") != -1) \{
216. resultresult = result.replaceAll("\\t", "");
217. \}
218. // 去除多余空格
219. while (result.indexOf(" ") != -1) \{
220. resultresult = result.replaceAll(" ", " ");
221. \}
222. // 去除全角空格
223. while (result.indexOf("　") != -1) \{
224. resultresult = result.replaceAll("　", "");
225. \}
226. return result;
227. \} else \{
228. return null;
229. \}
230. \}
231. 
232. // 使用HtmlParser组件去除内容中的HTML标签，得到纯文本内容
233. public static String getText(String content, String encode) \{
234. String result = content;
235. try \{
236. Parser parser = Parser.createParser(content, encode);
237. // 创建TextExtractingVisitor对象
238. TextExtractingVisitor visitor = new TextExtractingVisitor();
239. // 去除网页中的所有标签,提出纯文本内容
240. parser.visitAllNodesWith(visitor);
241. result = visitor.getExtractedText();
242. // System.out.println("\#\#\#去除HTML标签:" + result);
243. \} catch (Exception ex) \{
244. ex.printStackTrace();
245. \}
246. return result;
247. \}
248. 
249. // 查询网页包含某种标签的集合数组
250. @SuppressWarnings("unchecked")
251. public static Node\[\] getTagList(String htmlCode, String encode, Class t) \{
252. Node\[\] nodes = null;
253. try \{
254. Parser parser = Parser.createParser(htmlCode, encode);
255. ObjectFindingVisitor visitor = new ObjectFindingVisitor(t);
256. parser.visitAllNodesWith(visitor);
257. nodes = visitor.getTags();
258. \} catch (Exception e) \{
259. e.printStackTrace();
260. \}
261. return nodes;
262. \}
263. 
264. // 判断自定字符串是否符号某种正则规则
265. public static boolean isRegex(String isRegexString, String regexString) \{
266. boolean regexStatus = Pattern.matches(regexString, isRegexString);
267. return regexStatus;
268. \}
269. \}

运行结果:

图片上传有问题，大家可以运行一下查看结果。

希望各位多多指点！

转载于:https://blog.51cto.com/yaodian/751297