scrapy的真实使用

偏执的太偏执、 2022-07-15 07:26 171阅读 0赞

一、伪造头部和cookie
    1.HEADER 和 COOKIES 在 settings.py 中定义如下：
    
    HEADER={
        "Host": "www.zhihu.com",
        "Connection": "keep-alive",
        "Cache-Control": "max-age=0",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36",
        "Referer": "http://www.zhihu.com/people/raymond-wang",
        "Accept-Encoding": "gzip,deflate,sdch",
        "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4,zh-TW;q=0.2",
        }
    
    COOKIES={
        'checkcode':r'"$2a$10$9FVE.1nXJKq/F.nH62OhCevrCqs4skby2bC4IO6VPJITlc7Sh.NZa"',
        'c_c':r'a153f80493f411e3801452540a3121f7',
        '_ga':r'GA1.2.1063404131.1384259893',
        'zata':r'zhihu.com.021715f934634a988abbd3f1f7f31f37.470330',
        'q_c1':r'59c45c60a48d4a5f9a12a52028a9aee7|1400081868000|1400081868000',
        '_xsrf':r'2a7cf7208bf24dbda3f70d953e948135',
        'q_c0':r'"NmE0NzBjZTdmZGI4Yzg3ZWE0NjhkNjkwZGNiZTNiN2F8V2FhRTQ1QklrRjNjNGhMdQ==|1400082425|a801fc83ab07cb92236a75c87de58dcf3fa15cff"',
        '__utma':r'51854390.1063404131.1384259893.1400518549.1400522270.5',
        '__utmb':r'51854390.4.10.1400522270',
        '__utmc':r'51854390',
        '__utmz':r'51854390.1400513283.3.3.utmcsr=zhihu.com|utmccn=(referral)|utmcmd=referral|utmcct=/people/hallson',
        '__utmv':r'51854390.100-1|2=registration_date=20121016=1^3=entry_date=20121016=1'
    }
    
    
    2.调用
    # -*- coding:utf-8 -*-
    
    from scrapy.selector import Selector
    from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
    from scrapy.contrib.spiders import CrawlSpider, Rule
    from scrapy.http import Request,FormRequest
    
    from zhihu.settings import *
    
    class ZhihuLoginSpider(CrawlSpider):
        name = 'zhihulogin1'
        allowed_domains = ['zhihu.com']
        start_urls = ['http://www.zhihu.com/lookup/class/']
    
        rules = (
            Rule(SgmlLinkExtractor(allow=r'search/')),
            Rule(SgmlLinkExtractor(allow=r'')),
        )
    
        def __init__(self):
            self.headers =HEADER
            self.cookies =COOKIES
    
        def start_requests(self):
            for i, url in enumerate(self.start_urls):
                yield FormRequest(url, meta = {
       'cookiejar': i}, \
                                  headers = self.headers, \
                                  cookies =self.cookies,
                                  callback = self.parse_item)#jump to login page
    
        def parse_item(self, response):
            selector = Selector(response)
    
            urls = []
            for ele in selector.xpath('//ul/li[@class="suggest-item"]/div/a/@href').extract():
               urls.append(ele)
            print urls
    
    
    二、自动登录并采集某网站中的所有用户信息
    1）.重写 start_requests 方法，通过设置 FormRequest 的 formdata 参数，这里是 email 和 password，然后提交请求到 http://www.zhihu.com/login进行登陆，如果登陆成功之后，调用 after_login 回调方法。
    2）.在 after_login 方法中，逐个访问 start_urls 中的 url
    3）rules 中定义了一些正则匹配的 url 所对应的回调函数
    
    使用账户和密码进行登陆代码如下：
    
    # -*- coding:utf-8 -*-
    from scrapy.selector import Selector
    from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
    from scrapy.contrib.spiders import CrawlSpider, Rule
    from scrapy.http import Request,FormRequest
    
    import sys
    
    reload(sys)
    sys.setdefaultencoding('utf-8')
    
    host='http://www.zhihu.com'
    
    class ZhihuUserSpider(CrawlSpider):
        name = 'zhihu_user'
        allowed_domains = ['zhihu.com']
        start_urls = ["http://www.zhihu.com/lookup/people",]
    
        #使用rule时候，不要定义parse方法
        rules = (
            Rule(SgmlLinkExtractor(allow=("/lookup/class/[^/]+/?$", )), follow=True,callback='parse_item'),
            Rule(SgmlLinkExtractor(allow=("/lookup/class/$", )), follow=True,callback='parse_item'),
            Rule(SgmlLinkExtractor(allow=("/lookup/people", )),  callback='parse_item'),
       )
    
        def __init__(self,  *a,  **kwargs):
            super(ZhihuLoginSpider, self).__init__(*a, **kwargs)
    
        #设置账号密码进行自动登录
        def start_requests(self):
            return [FormRequest(
                "http://www.zhihu.com/login",
                formdata = {
       'email':'XXXXXX',
                            'password':'XXXXXX'
                },
                callback = self.after_login
            )]
    
        #逐个访问start_urls 中的 url
        def after_login(self, response):
            for url in self.start_urls:
                yield self.make_requests_from_url(url)
    
        def parse_item(self, response):
            selector = Selector(response)
            for link in selector.xpath('//div[@id="suggest-list-wrap"]/ul/li/div/a/@href').extract():
                #link  ===> /people/javachen
                yield Request(host+link+"/about", callback=self.parse_user)
    
        #提取用户信息，通过提取关注数，从而提取关注列表并从中使用parse_follow_url分析处更多关注列表
        def parse_user(self, response): 
            selector = Selector(response)
            user = ZhihuUserItem()
            user['_id']=user['username']=response.url.split('/')[-2]
            user['url']= response.url
            user['nickname'] = ''.join(selector.xpath("//div[@class='title-section ellipsis']/a[@class='name']/text()").extract())
            user['location'] = ''.join(selector.xpath("//span[@class='location item']/@title").extract())
            user['industry'] = ''.join(selector.xpath("//span[@class='business item']/@title").extract())
            user['sex'] = ''.join(selector.xpath('//div[@class="item editable-group"]/span/span[@class="item"]/i/@class').extract()).replace("zg-icon gender ","")
            user['description'] = ''.join(selector.xpath("//span[@class='description unfold-item']/span/text()").extract()).strip().replace("\n",'')
            user['view_num'] = ''.join(selector.xpath("//span[@class='zg-gray-normal']/strong/text()").extract())
            user['update_time'] = str(datetime.now())
            #抓取用户信息，此处省略更多代码
    
        #获取到用户的关注数 followee_num后，继续采集该用户所有的关注列表
            _xsrf = ''.join(selector.xpath('//input[@name="_xsrf"]/@value').extract())
            hash_id = ''.join(selector.xpath('//div[@class="zm-profile-header-op-btns clearfix"]/button/@data-id').extract())
    
            num = int(followee_num) if followee_num else 0
        page_num = num/20
        page_num += 1 if num%20 else 0
        for i in xrange(page_num):
                params = json.dumps({
       "hash_id":hash_id,"order_by":"created","offset":i*20})
                payload = {
       "method":"next", "params": params, "_xsrf":_xsrf}
                yield Request("http://www.zhihu.com/node/ProfileFolloweesListV2?"+urlencode(payload), callback=self.parse_follow_url)
    
        #将用户关注列表中的链接转化为用户信息采集链接，并继续采集，并回调parse_user提取用户信息
        def parse_follow_url(self, response):
            selector = Selector(response)
    
            for link in selector.xpath('//div[@class="zm-list-content-medium"]/h2/a/@href').extract():
                #link  ===> http://www.zhihu.com/people/peng-leslie-97
                username_tmp = link.split('/')[-1]
                if username_tmp in self.user_names:
                    print 'GET:' + '%s' % username_tmp
                    continue
    
                yield Request(link+"/about", callback=self.parse_user)