【scrapy ip代理池】如何解决ip被限制的问题

太过爱你忘了你带给我的痛 2021-10-14 02:24 418阅读 0赞

**1、首先写一个脚本proxies.py 自动获取代理ip**

\# \*-\* coding:utf-8 \*-\*  
import requests  
from bs4 import BeautifulSoup  
import lxml  
from multiprocessing import Process, Queue  
import random  
import json  
import time  
import requests

class Proxies(object):

"""docstring for Proxies"""  
    def \_\_init\_\_(self, page=3):  
        self.proxies = \[\]  
        self.verify\_pro = \[\]  
        self.page = page  
        self.headers = \{  
        'Accept': '\*/\*',  
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',  
        'Accept-Encoding': 'gzip, deflate, sdch',  
        'Accept-Language': 'zh-CN,zh;q=0.8'  
        \}  
        self.get\_proxies()  
        self.get\_proxies\_nn()

def get\_proxies(self):  
        page = random.randint(1,10)  
        page\_stop = page + self.page  
        while page < page\_stop:  
            url = 'http://www.xicidaili.com/nt/%d' % page  
            html = requests.get(url, headers=self.headers).content  
            soup = BeautifulSoup(html, 'lxml')  
            ip\_list = soup.find(id='ip\_list')  
            for odd in ip\_list.find\_all(class\_='odd'):  
                protocol = odd.find\_all('td')\[5\].get\_text().lower()+'://'  
                self.proxies.append(protocol + ':'.join(\[x.get\_text() for x in odd.find\_all('td')\[1:3\]\]))  
            page += 1

def get\_proxies\_nn(self):  
        page = random.randint(1,10)  
        page\_stop = page + self.page  
        while page < page\_stop:  
            url = 'http://www.xicidaili.com/nn/%d' % page  
            html = requests.get(url, headers=self.headers).content  
            soup = BeautifulSoup(html, 'lxml')  
            ip\_list = soup.find(id='ip\_list')  
            for odd in ip\_list.find\_all(class\_='odd'):  
                protocol = odd.find\_all('td')\[5\].get\_text().lower() + '://'  
                self.proxies.append(protocol + ':'.join(\[x.get\_text() for x in odd.find\_all('td')\[1:3\]\]))  
            page += 1

def verify\_proxies(self):  
        \# 没验证的代理  
        old\_queue = Queue()  
        \# 验证后的代理  
        new\_queue = Queue()  
        print ('verify proxy........')  
        works = \[\]  
        for \_ in range(15):  
            works.append(Process(target=self.verify\_one\_proxy, args=(old\_queue,new\_queue)))  
        for work in works:  
            work.start()  
        for proxy in self.proxies:  
            old\_queue.put(proxy)  
        for work in works:  
            old\_queue.put(0)  
        for work in works:  
            work.join()  
        self.proxies = \[\]  
        while 1:  
            try:  
                self.proxies.append(new\_queue.get(timeout=1))  
            except:  
                break  
        print ('verify\_proxies done!')

def verify\_one\_proxy(self, old\_queue, new\_queue):  
        while 1:  
            proxy = old\_queue.get()  
            if proxy == 0:break  
            protocol = 'https' if 'https' in proxy else 'http'  
            proxies = \{protocol: proxy\}  
            try:  
                if requests.get('http://www.baidu.com', proxies=proxies, timeout=2).status\_code == 200:  
                    print ('success %s' % proxy)  
                    new\_queue.put(proxy)  
            except:  
                print ('fail %s' % proxy)

if \_\_name\_\_ == '\_\_main\_\_':  
    a = Proxies()  
    a.verify\_proxies()  
    print (a.proxies)  
    proxie = a.proxies  
    with open('proxies.txt', 'a') as f:  
       for proxy in proxie:  
             f.write(proxy+'\\n')

**2、接下来修改代理文件middlewares.py的内容为如下:**

import random    
import scrapy    
from scrapy import log

\# logger = logging.getLogger()

class ProxyMiddleWare(object):    
    """docstring for ProxyMiddleWare"""    
    def process\_request(self,request, spider):    
        '''对request对象加上proxy'''    
        proxy = self.get\_random\_proxy()    
        print("this is request ip:"+proxy)    
        request.meta\['proxy'\] = proxy

def process\_response(self, request, response, spider):    
        '''对返回的response处理'''    
        \# 如果返回的response状态不是200，重新生成当前request对象    
        if response.status != 200:    
            proxy = self.get\_random\_proxy()    
            print("this is response ip:"+proxy)    
            \# 对当前reque加上代理    
            request.meta\['proxy'\] = proxy     
            return request    
        return response

def get\_random\_proxy(self):    
        '''随机从文件中读取proxy'''    
        while 1:    
            with open('G:\\\\Scrapy\_work\\\\myproxies\\\\myproxies\\\\proxies.txt', 'r') as f:    
                proxies = f.readlines()    
            if proxies:    
                break    
            else:    
                time.sleep(1)    
        proxy = random.choice(proxies).strip()    
        return proxy    
  
**3、修改下settings文件**

DOWNLOADER\_MIDDLEWARES = \{    
   'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware':None,    
     'youx.middlewares.ProxyMiddleWare':125,    
     'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware':None    
\}

或者

DOWNLOADER\_MIDDLEWARES = \{ 'haodf.middlewares.ProxyMiddleWare':400 \}

想要ip应用起来，还要在配置文件 settings 中添加 DOWNLOADER\_MIDDLEWARES = \{ 'demo.proxy.proxMiddleware':400 \} 这里的demo是工程的名字，proxy是py文件的名,proxMiddleware是类的名字