scrapy中selenium模拟点击下一页代码

冷不防 2022-05-24 13:09 153阅读 0赞

class ZufangDownloaderMiddleware(object):
        # Not all methods need to be defined. If a method is not defined,
        # scrapy acts as if the downloader middleware does not modify the
        # passed objects.
    
        @classmethod
        def from_crawler(cls, crawler):
            # This method is used by Scrapy to create your spiders.
            s = cls()
            crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
            return s
    
        def process_request(self, request, spider):
            click_page_url = "http://www.XXX.com"
            print(request.url)
            initcnt = 0
            trycnt = 0
    
            while (initcnt < 3):
                try:
                    initcnt += 1
                    if click_page_url in request.url:
                        options = webdriver.FirefoxOptions()
                        options.add_argument('--headless')
                        options.add_argument('--disable-gpu')
                        driver = webdriver.Firefox(firefox_options=options)
    
                        while (trycnt < 3) :
                            try:
                                trycnt += 1
                                driver.get(request.url)
                                driver.implicitly_wait(5)
                                time.sleep(5)
                                print(driver.title)
    
                                look_more = "//a[@id='goNextPeriod']"
                                driver.find_element_by_xpath(look_more).click()
                                time.sleep(5)
                                print(driver.current_url)
    
                                urlchangecnt = 0
                                while (urlchangecnt < 3) :
                                    urlchangecnt += 1
                                    if request.url in driver.current_url:
                                        driver.find_element_by_xpath(look_more).click()
                                        time.sleep(5+urlchangecnt*2)
                                        print(driver.current_url)
                                    else:
                                        break
    
                                true_page = driver.page_source
                                # print(true_page)
    
                                new_url = {
      'new_url': driver.current_url}
                                driver.close()
    
                                return HtmlResponse(request.url, body=true_page, headers=new_url, encoding='utf-8', request=request)
    
                            except:
                                print("get news data failed ")
                    else:
                        return None
                except:
                    print("webdriver init failed ")
            return None