java实现爬虫爬取京东手机页面

喜欢ヅ旅行 2022-02-04 00:59 480阅读 1赞

java实现爬虫利用httpclient获取页面数据,再用jsoup解析获取数据,在此页面的分析不作过多赘述,读者自行分析.

1.首页输入手机,观察url参数,将其中”&page=”提取到最后,便于拼接页码参数;

2.查看网页源代码,观看到商品列表标签,所以第一步,获取商品列表,Elements spuEles = doc.select( “div#J_goodsList>ul>li” );第二步,获取spu,long spu = Long.parseLong( spuEle.attr( “data-spu” ) );第三步,获取sku列表,Elements skuEles = spuEle.select( “li.ps-item” );第四步,获取单个sku,long sku = Long.parseLong( skuEle.select( “[data-sku]“ ).attr( “data-sku” ) );之后的操作就都是通过这个sku来操作的;

3.代码:

(1)添加依赖

  1. <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
  2.         <dependency>
  3.             <groupId>org.apache.httpcomponents</groupId>
  4.             <artifactId>httpclient</artifactId>
  5.         </dependency>
  6.         <!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
  7.         <dependency>
  8.             <groupId>org.jsoup</groupId>
  9.             <artifactId>jsoup</artifactId>
  10.             <version>1.11.3</version>
  11.         </dependency>
  12.         <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
  13.         <dependency>
  14.             <groupId>org.apache.commons</groupId>
  15.             <artifactId>commons-lang3</artifactId>
  16.             <version>3.4</version>
  17.         </dependency>
  18. <!-- jpa -->
  19.         <dependency>
  20.             <groupId>org.springframework.boot</groupId>
  21.             <artifactId>spring-boot-starter-data-jpa</artifactId>
  22.         </dependency>

数据库等其他依赖就不贴了;

(2)工具类封装

  1. @Component
  2. public class HttpUtils {
  4.     private PoolingHttpClientConnectionManager cm;
  6.     public HttpUtils() {
  7.         this.cm = new PoolingHttpClientConnectionManager();
  8.         // 设置最大连接数
  9.         this.cm.setMaxTotal( 100 );
  10.         // 设置每个主机的最大连接数
  11.         this.cm.setDefaultMaxPerRoute( 10 );
  12.     }
  14.     // 根据请求地址下载页面数据
  15.     public String doGetHtml( String url ) {
  16.         // 获取httpclient对象
  17.         CloseableHttpClient httpClient = HttpClients.custom()
  18.                                                     .setConnectionManager( this.cm )
  19.                                                     .build();
  20.         // 设置httpGet请求对象,设置url地址
  21.         HttpGet httpGet = new HttpGet( url );
  22.         httpGet.addHeader( "user-agent", "Mozilla/5.0" );
  23.         // 设置请求信息
  24.         httpGet.setConfig( this.getConfig() );
  25.         CloseableHttpResponse response = null;
  26.         try {
  27.             // 使用httpclient发起请求,回去响应
  28.             response = httpClient.execute( httpGet );
  29.             // 解析响应.返回结果
  30.             if ( response.getStatusLine()
  31.                          .getStatusCode() == 200 ) {
  32.                 // 判断响应体Entity是否不为空,如果不为空就可以使用EntityUtils
  33.                 if ( response.getEntity() != null ) {
  34.                     String content = EntityUtils.toString( response.getEntity(), "utf-8" );
  35.                     return content;
  36.                 }
  37.             }
  38.         } catch( Exception e ) {
  39.             e.printStackTrace();
  40.         } finally {
  41.             try {
  42.                 // 关闭response
  43.                 if ( response != null ) {
  44.                     response.close();
  45.                 }
  46.             } catch( IOException e ) {
  47.                 e.printStackTrace();
  48.             }
  49.         }
  50.         // 返回空字符串
  51.         return "";
  52.     }
  54.     /**
  55.      * //下载图片名称
  56.      * 
  57.      * @param url
  58.      * @return
  59.      */
  60.     public String doGetImage( String url ) {
  61.         // 获取httpclient对象
  62.         CloseableHttpClient httpClient = HttpClients.custom()
  63.                                                     .setConnectionManager( this.cm )
  64.                                                     .build();
  65.         // 设置httpGet请求对象,设置url地址
  66.         HttpGet httpGet = new HttpGet( url );
  67.         httpGet.addHeader( "user-agent", "Mozilla/5.0" );
  68.         // 设置请求信息
  69.         httpGet.setConfig( this.getConfig() );
  70.         CloseableHttpResponse response = null;
  71.         try {
  72.             // 使用httpclient发起请求,回去响应
  73.             response = httpClient.execute( httpGet );
  74.             // 解析响应.返回结果
  75.             if ( response.getStatusLine()
  76.                          .getStatusCode() == 200 ) {
  77.                 // 判断响应体Entity是否不为空,如果不为空就可以使用EntityUtils
  78.                 if ( response.getEntity() != null ) {
  79.                     // 下载图片
  80.                     // 获取图片的后缀
  81.                     String extName = url.substring( url.lastIndexOf( "." ) );
  82.                     // 创建图片名,重命名图片
  83.                     String picName = UUID.randomUUID()
  84.                                          .toString() +
  85.                                      extName;
  86.                     // 下载图片
  87.                     // 声明OutPutStream
  88.                     OutputStream outPutStream = new FileOutputStream( new File( "D:\\driver\\images\\" + picName ) );
  89.                     response.getEntity()
  90.                             .writeTo( outPutStream );
  91.                     // 返回图片名称
  92.                     return picName;
  93.                 }
  94.             }
  95.         } catch( Exception e ) {
  96.             e.printStackTrace();
  97.         } finally {
  98.             try {
  99.                 // 关闭response
  100.                 if ( response != null ) {
  101.                     response.close();
  102.                 }
  103.             } catch( IOException e ) {
  104.                 e.printStackTrace();
  105.             }
  106.         }
  107.         // 如果下载失败,返回空字符串
  108.         return "";
  109.     }
  111.     // 设置请求信息
  112.     private RequestConfig getConfig() {
  113.         RequestConfig config = RequestConfig.custom()
  114.                                             .setConnectTimeout( 1000 ) // 创建连接的最长时间
  115.                                             .setConnectionRequestTimeout( 500 ) // 获取连接的最长时间
  116.                                             .setSocketTimeout( 10000 ) // 数据传输最长时间
  117.                                             .build();
  118.         return config;
  119.     }
  120. }

(3)dao层,service层以及实体类

  1. public interface ItemDao extends JpaRepository<Item, Long> {
  3. }
  5. @Service
  6. public class ItemService {
  8.     @Autowired
  9.     private ItemDao itemDao;
  11.     @Transactional
  12.     public void save( Item item ) {
  13.         this.itemDao.save( item );
  14.     }
  16.     public List<Item> findAll( Item item ) {
  17.         // 声明查询条件
  18.         Example<Item> example = Example.of( item );
  19.         // 根据查询条件进行查询数据
  20.         List<Item> list = this.itemDao.findAll( example );
  21.         return list;
  22.     }
  23. }
  25. @Entity
  26. @Table( name = "jd_item" )
  27. @Data
  28. public class Item {
  30.     @Id
  31.     @GeneratedValue( strategy = GenerationType.IDENTITY )
  32.     private Long   id;
  34.     private Long   spu;
  36.     private Long   sku;
  38.     private String title;
  40.     private Double price;
  42.     private String pic;
  44.     private String url;
  46.     private Date   created;
  48.     private Date   updated;
  49. }

(4)定时任务

  1. @Component
  2. public class ItemTask {
  4.     @Autowired
  5.     private HttpUtils                 httpUtils;
  7.     @Autowired
  8.     private ItemService               itemService;
  10.     @Autowired
  11.     private static final ObjectMapper MAPPER = new ObjectMapper();
  13.     // 当下载任务完成后,间隔多长时间进行下一次的任务
  14.     @Scheduled( fixedDelay = 100 * 1000 )
  15.     public void itemTask() throws Exception {
  16.         // 声明需要解析的初始地址
  17.         String url = "https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&suggest=1.def.0.V13--12%2C20%2C&wq=shouji&cid2=653&cid3=655&s=59&click=0&page=";
  18.         // 遍历页面
  19.         for( int i = 1; i < 10; i = i + 2 ) {
  20.             String html = httpUtils.doGetHtml( url + i );
  21.             // 解析页面,获取商品数据并存储
  22.             this.parse( html );
  23.         }
  24.         System.out.println( "手机数据抓取完成!" );
  25.     }
  27.     // 解析页面,获取商品数据并存储
  28.     private void parse( String html ) throws Exception {
  29.         // 解析html,获取dom对象
  30.         Document doc = Jsoup.parse( html );
  31.         // 获取商品列表
  32.         Elements spuEles = doc.select( "div#J_goodsList>ul>li" );
  33.         for( Element spuEle : spuEles ) {
  34.             // 获取spu
  35.             long spu = Long.parseLong( spuEle.attr( "data-spu" ) );
  36.             // 获取sku列表
  37.             Elements skuEles = spuEle.select( "li.ps-item" );
  38.             for( Element skuEle : skuEles ) {
  39.                 // 获取sku
  40.                 long sku = Long.parseLong( skuEle.select( "[data-sku]" )
  41.                                                  .attr( "data-sku" ) );
  42.                 // 根据sku查询商品数据
  43.                 Item item = new Item();
  44.                 item.setSku( sku );
  45.                 List<Item> list = this.itemService.findAll( item );
  46.                 if ( list.size() > 0 ) {
  47.                     // 如果商品存在,就进行下一个循环,该商品不保存,因为已存在
  48.                     continue;
  49.                 }
  50.                 // 设置商品的spu
  51.                 item.setSpu( spu );
  52.                 // 获取商品详情的url
  53.                 String itemUrl = "https://item.jd.com/" + sku + ".html";
  54.                 item.setUrl( itemUrl );
  55.                 // 获取商品的图片
  56.                 String picUrl = "https:" + skuEle.select( "img[data-sku]" )
  57.                                                  .first()
  58.                                                  .attr( "data-lazy-img" );
  59.                 picUrl = picUrl.replace( "/n9", "/n1" );
  60.                 String picName = this.httpUtils.doGetImage( picUrl );
  61.                 item.setPic( picName );
  62.                 // 获取商品的价格
  63.                 String priceJson = this.httpUtils.doGetHtml( "https://p.3.cn/prices/mgets?skuIds=J_" + sku );
  64.                 double price = MAPPER.readTree( priceJson )
  65.                                      .get( 0 )
  66.                                      .get( "p" )
  67.                                      .asDouble();
  68.                 item.setPrice( price );
  69.                 // 获取商品的标题
  70.                 String itemInfo = this.httpUtils.doGetHtml( item.getUrl() );
  71.                 String title = Jsoup.parse( itemInfo )
  72.                                     .select( "div.sku-name" )
  73.                                     .text();
  74.                 item.setTitle( title );
  75.                 item.setCreated( new Date() );
  76.                 item.setUpdated( item.getCreated() );
  77.                 // 保存商品数据到数据库
  78.                 this.itemService.save( item );
  79.             }
  80.         }
  81.     }
  82. }

注意事项,一开始我httpclient没有添加header,导致获取到的html都是跳转到登录页面,后面模拟浏览器之后OK了

发表评论

表情：
评论列表 （有 0 条评论，480人围观）

还没有评论，来说两句吧...

相关阅读