评论爬虫小样

╰+攻爆jí腚メ 2024-04-20 08:39 76阅读 0赞

随着反爬手段的推陈出新，使用爬虫框架被反爬的几率也越来越高了， 于是我尝试使用最呆笨的方式爬取试了一下，居然可以，在此做下记录。

# encoding=utf-8
    import re
    import os
    import time
    import json
    import pathlib
    import requests
    import logging
    import random
    from pymongo import MongoClient
    
    
    # 设置日志的输出样式
    logging.basicConfig(level=logging.INFO,
                        format='[%(asctime)-15s] [%(levelname)8s] [%(name)10s ] - %(message)s (%(filename)s:%(lineno)s)',
                        datefmt='%Y-%m-%d %T'
                        )
    logger = logging.getLogger(__name__)
    
    # 启动爬虫前需先修改下面三项的值
    itemId= 582343703210
    base_url = "https://rate.tmall.com/list_detail_rate.htm?itemId={}&spuId=1112124124&sellerId=430490406&order=3&currentPage={}"
    cookie= 'cna=84DqFV4SPyYCATo9kTLfT6u+; t=aa42477f58c7f2322f00dfb5a1eb3ecc; _tb_token_=7e51fd1e5e1e7; cookie2=156089853e8f3a6eeb0f7920d1963fc3; _med=dw:1600&dh:900&pw:1600&ph:900&ist:0; otherx=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0; x=__ll%3D-1%26_ato%3D0; swfstore=192694; tk_trace=1; _m_h5_tk=fefe9396dafd34344a5209f356fa5219_1568970404033; _m_h5_tk_enc=95eb1bc833877a790160e51ac2b95c60; enc=HPXzwVBtnTh2ZKD7IdgorhLo07qNH2rA9jqbXScJDYdMLIFeET66f7y07GgZfiMfpKBC%2BItvWd2MLhSwCstmeA%3D%3D; tt=login.tmall.com; res=scroll%3A990*6474-client%3A924*772-offset%3A924*6474-screen%3A1600*900; whl=-1%260%260%260; cq=ccp%3D1; pnm_cku822=098%23E1hv9vvUvbpvUvCkvvvvvjiPRFLy0jDCRscvzjthPmPwtjnmR2sw0jECnLdUzjYRiQhvCvvvpZptvpvhvvCvpvGCvvpvvPMMmphvLvm4vvvaaB4AdBDQbNLt%2Bu6XjC61D76Xj8TJEcqUz8gL%2BulAp57Q%2Bu6XeC69%2FX7rVBAKfvDrAEuK53rsxXxreEeKHkx%2FAWmK5dot2e%2FivpvUvvCCb2cDLs8EvpvVvpCmpYsZKphv8vvvphvvvvvvvvC2q9vv9OIvvhOVvvmCp9vvB9OvvUhKvvCVC9vv9ogCvpvVvUCvpvvv; l=cBjfAG8qqFtnScxbBOCwNQKXiCQT9IRAguSJGwSBi_5CB6LsfqQOk_LFbFp6cjWdtLLB4-YhSFe9-etki0ILNztqTtrR.; isg=BJSUQtNutcPQJiHsng_MgZdGZdKq_ej41L3UzC51G5-iGTRjVv6lZ_mbGVEk4fAv; dnk=pengjunlee1; uc1=tag=8&lng=zh_CN&cookie15=V32FPkk%2Fw0dUvg%3D%3D&cookie14=UoTaEC%2BQ14Cucg%3D%3D&cookie21=WqG3DMC9Eman&pas=0&existShop=false&cookie16=WqG3DMC9UpAPBHGz5QBErFxlCA%3D%3D; uc3=vt3=F8dByuK3QCZUfvQCEBM%3D&id2=UUphy%2FZ7Cdwys6i9Hw%3D%3D&nk2=E6EQ1CLKS1V0cko%3D&lg2=URm48syIIVrSKA%3D%3D; tracknick=pengjunlee1; lid=pengjunlee1; _l_g_=Ug%3D%3D; uc4=nk4=0%40EbhmhLlrKXZUZrdwvqX9l9vjDvaQPA%3D%3D&id4=0%40U2grEJGEPwwA5lpVHyA%2BGBdaEx29RDor; unb=2201414542619; lgc=pengjunlee1; cookie1=VvuEIF254OaIGYBx5CMIrukTXbnCoYs81mNl4ECxqes%3D; login=true; cookie17=UUphy%2FZ7Cdwys6i9Hw%3D%3D; _nk_=pengjunlee1; sg=196; csg=96ab7a84'
    
    referer = 'https://detail.tmall.com/item.htm?id={}'.format(itemId)
    headers = {
        'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36',
        'Referer' : referer,
        'Cookie' : cookie,
        'accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
        'accept-encoding' : 'gzip, deflate, br',
        'accept-language' : 'zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7'
    }
    
    # 为每一个商品创建一个单独的文件夹用来存放爬到的内容
    fpath = 'D:\\scrapy\\new_tmall\\{}'.format(itemId)
    if not os.path.exists(fpath):
        os.makedirs(fpath)
    
    rate_pattern = re.compile(r'jsonp\d+$(.*?)$$', re.S)
    session = requests.Session()
    pageCount = 0
    currentPage = 1
    # 一次最多爬取15页，防止被封
    requests_count = 15
    
    while ( currentPage <= pageCount or pageCount == 0) and requests_count>0:
    
        logger.info("开始爬取第 {} 页".format(currentPage))
        pagePath = pathlib.Path('D:\\scrapy\\new_tmall\\{}\\{}.txt'.format(itemId,currentPage))
        if pagePath.exists():
            logger.info("使用本地文件。。。")
            with open('D:\\scrapy\\new_tmall\\{}\\{}.txt'.format(itemId,currentPage), 'r', encoding='utf-8') as f:
                json_str = f.read()
            ret_str = rate_pattern.search(json_str)
        else:
            logger.info("发送网络请求。。。")
            url = base_url.format(itemId, currentPage)
            time.sleep(5+random.randint(0,5))
            try :
                requests_count = requests_count-1
                response = session.get(url, headers=headers)
            except BaseException as e:
                logger.info(e)
                break
            else:
                json_str = response.content.decode()
                ret_str = rate_pattern.search(json_str)
                if ret_str:
                    with open('D:\\scrapy\\new_tmall\\{}\\{}.txt'.format(itemId,currentPage), 'w', encoding='utf-8') as f:
                        f.write(response.content.decode())
                else:
                    break
        if ret_str:
            # 连接Mongodb数据库
            m = MongoClient(host="172.16.250.238", port=27017)
            test_db = m["test"]
            db = test_db['tmallRateEntity']
    
            json_obj = json.loads(ret_str.group(1))
            rate_list = json_obj['rateDetail']['rateList']
            with open('{}.txt'.format(itemId), 'a', encoding='utf-8') as f:
                for rate in rate_list:
                    o = {}
                    o['rateContent'] = rate['rateContent']
                    o['rateDate'] = rate['rateDate']
                    o['_id'] = rate['id']
                    o['goodsId'] = itemId
                    o['pics'] = rate['pics']
                    o['anony'] = rate['anony']
                    o['userVipLevel'] = rate['userVipLevel']
                    o['goldUser'] = rate['goldUser']
                    o['displayUserNick'] = rate['displayUserNick']
                    o['appendComment'] = rate['appendComment']
                    ret = db.find_one({'_id': o['_id']})
                    if ret is None:
                        db.insert_one(o)
            if pageCount == 0:
                pageCount = json_obj['rateDetail']['paginator']['lastPage']
        logger.info("总共 {} 页".format(pageCount))
        currentPage = currentPage + 1