在爬虫框架scrapy中使用selenium

左手的ㄟ右手 2023-06-25 10:06 9阅读 0赞

##### 1. wangyi.py #####

import scrapy
    from selenium import webdriver
    
    
    ''' 在scrapy中使用selenium的编码流程: 1.在spider的构造方法中创建一个浏览器对象(作为当前spider的一个属性) 2.重写spider的一个方法closed(self,spider),在该方法中执行浏览器关闭的操作 3.在下载中间件的process_response方法中,通过spider参数获取浏览器对象 4.在中间件的process_response中定制基于浏览器自动化的操作代码(获取动态加载出来的页面源码数据) 5.实例化一个响应对象,且将page_source返回的页面源码封装到该对象中 6.返回该新的响应对象 '''
    
    class TestSpider(scrapy.Spider):
        name = 'wangyi'
        # allowed_domains = ['www.xxx.com']
        start_urls = ['http://war.163.com/']
        def __init__(self):
            self.browser = webdriver.Chrome()
    
        def parse(self, response):
            div_list = response.xpath('//div[@class="data_row news_article clearfix "]')
            for div in div_list:
                title = div.xpath('.//div[@class="news_title"]/h3/a/text()').extract_first()
                print(title)
    
        def closed(self, spider):
            print('bro has been closed')
            self.browser.quit()

##### 2. middlewares.py #####

from scrapy import signals
    from scrapy.http import HtmlResponse
    from time import sleep
    
    
    class WangyiproDownloaderMiddleware(object):
        # Not all methods need to be defined. If a method is not defined,
        # scrapy acts as if the downloader middleware does not modify the
        # passed objects.
    
        @classmethod
        def from_crawler(cls, crawler):
            # This method is used by Scrapy to create your spiders.
            s = cls()
            crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
            return s
    
        def process_request(self, request, spider):
            # Called for each request that goes through the downloader
            # middleware.
    
            # Must either:
            # - return None: continue processing this request
            # - or return a Response object
            # - or return a Request object
            # - or raise IgnoreRequest: process_exception() methods of
            # installed downloader middleware will be called
            return None
    
        def process_response(self, request, response, spider):
            # Called with the response returned from the downloader.
    
            # Must either;
            # - return a Response object
            # - return a Request object
            # - or raise IgnoreRequest
            print('即将返回一个新的响应对象!!!')
            #如何获取动态加载出来的数据
            bro = spider.bro
            bro.get(url=request.url)
            sleep(3)
            #包含了动态加载出来的新闻数据
            page_text = bro.page_source
            sleep(3)
            return HtmlResponse(url=spider.bro.current_url,body=page_text,encoding='utf-8',request=request)
    
        def process_exception(self, request, exception, spider):
            # Called when a download handler or a process_request()
            # (from other downloader middleware) raises an exception.
    
            # Must either:
            # - return None: continue processing this exception
            # - return a Response object: stops process_exception() chain
            # - return a Request object: stops process_exception() chain
            pass
    
        def spider_opened(self, spider):
            spider.logger.info('Spider opened: %s' % spider.name)

##### 3. 在中间件判断是否需要使用selenium来爬取内容的版本 #####

from scrapy.http import HtmlResponse    
        #参数介绍：
        #拦截到响应对象（下载器传递给Spider的响应对象）
        #request：响应对象对应的请求对象
        #response：拦截到的响应对象
        #spider：爬虫文件中对应的爬虫类的实例
        def process_response(self, request, response, spider):
            #响应对象中存储页面数据的篡改
            if request.url in['http://news.163.com/domestic/','http://news.163.com/world/','http://news.163.com/air/','http://war.163.com/']:
                spider.bro.get(url=request.url)
                js = 'window.scrollTo(0,document.body.scrollHeight)'
                spider.bro.execute_script(js)
                time.sleep(2)  #一定要给与浏览器一定的缓冲加载数据的时间
                #页面数据就是包含了动态加载出来的新闻数据对应的页面数据
                page_text = spider.bro.page_source
                #篡改响应对象
                return HtmlResponse(url=spider.bro.current_url,body=page_text,encoding='utf-8',request=request)
            else:
                return response