爬虫Spider--爬取京东某产品的评价

喜欢ヅ旅行 2022-04-02 07:20 256阅读 0赞

本篇博客提供了4种方式，有简略版仅能完成要求却简陋，也有较为完整的方式

# -*- coding:utf-8 -*-
    import re
    import urllib2
    import json
    import sys
    
    if sys.getdefaultencoding() != 'utf-8':
        reload(sys)
        sys.setdefaultencoding('utf-8')
    class JDSpider:
        def loadPage(self):
            url = "https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv566&productId=100001906474&score=0&sortType=5&page=1&pageSize=10&isShadowSku=0&rid=0&fold=1"
            user_agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
            headers = {'User-Agent': user_agent}
            req = urllib2.Request(url,headers=headers)
            response = urllib2.urlopen(req)
            html = response.read()
            pattern = re.compile('"content":"(.*?)".*?"nickname":"(.*?)"')
            # print html.decode("gbk")
            # pattern = re.compile(r',"creationTime"(.*?)"nickname":')
            # html = pattern.sub(r'', html)
            # print html
            # pattern = re.compile(r'"content":"(.*?)""(.*?)"')
            item_list = pattern.findall(html)
            print item_list
            for item in item_list:
                self.writetoFile(item)
        # def printPage(self,item_list,page):
        #     print item_list
        #     print "=====爬取第%d页===="%page
        #     for item in item_list:
        #         self.writetoFile(item)
        def writetoFile(self,test):
            with open("d:/124/jindong1.txt", 'a+')as myFile:
                json.dump(test, myFile, ensure_ascii=False, encoding='utf-8')
                # myFile.write(str(test).decode("utf-8"))
                myFile.write("\n-------------------------------------\n")
                myFile.close()
        # def doWork(self):
        #     while self.enable:
        #         try:
        #             item_list = self.loadPage(self.page)
        #         except urllib2.URLError,e:
        #             print e.reason
        #             continue
        #             self.printPage(item_list,self.page)
        #             self.page+=1
        #             print "按回车继续"
        #             print "输入quit退出"
        #             command = raw_input()
        #             if(command=="quit"):
        #                 break
    if __name__ == '__main__':
        """
            ======================
                京东评论爬虫
            ======================
        """
        #定义一个JDSpider对象
        mySpider = JDSpider()
        mySpider.loadPage()

# -*- coding:utf-8 -*-
    import urllib2
    import json
    import sys
    
    class Comment:
        def Commets(self):
            reload(sys)
            sys.setdefaultencoding('utf-8')
            f=open('02.txt', 'w')
            for i in range(0, 10):
                url='https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv562&productId=100001906474&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1' + str(
                 i) + '&pageSize=10&isShadowSku=0&fold=1'
    #             爬多页
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            html = response.read().decode('GBK')
    
            html = html.replace('fetchJSON_comment98vv562(', '')
            html = html.replace(');', '')
    #         去掉多余字符
            b = json.loads(html)
            print b
            for k in b['comments']:
                content = k["content"].encode('utf-8')
                self.writeToFile(content)
                f.write(k["content"].encode('utf-8') + '\n')
                referenceName = k["nickname"].encode('utf-8')
                self.writeToFile(referenceName)
                f.write(k["nickname"].encode('utf-8') + '\n')
                referenceTime = k["referenceTime"].encode('utf-8') + '\n\n'
                self.writeToFile(referenceTime)
    
        def writeToFile(self, text):
    #         @brief 将数据追加进文件内容
    #         @param text文件内容
            with open("d:/124/jd.txt", 'a') as myFile:
                myFile.write(text)
                myFile.write("\n-----------------------------------------------")
                myFile.close()
    
    if __name__ == '__main__':
        Comment = Comment()
        Comment.Commets()

# - * - coding: UTF-8 - * -
    import urllib2
    import json
    import sys
    
    reload(sys)
    sys.setdefaultencoding('utf8')
    
    f = open('01.txt', 'w')
    for i in range(0, 10):
        url = 'https://sclub.jd.com/comment/productPageComments.action?cal lback=fetchJSON_comment98vv562&productId=100001906474&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1' + str(i) + '&pageSize=10&isShadowSku=0&fold=1'
        # 实现爬多页
        print url
        request = urllib2.Request(url)
        response = urllib2.urlopen(request)
        html = response.read().decode('GBK')
    
        html = html.replace('fetchJSON_comment98vv562(', '')
        html = html.replace(');', '')
        # 去掉多余的字符
        b = json.loads(html)
    
        for k in b['comments']:
    
            content = k["content"].encode('utf-8')
            print content
            f.write(k["content"].encode('utf-8') + '\n')
            referenceName = k["referenceName"].encode('utf-8')
            print referenceName
            f.write(k["referenceName"].encode('utf-8') + '\n')
            referenceTime = k["referenceTime"].encode('utf-8')
            print referenceTime
            f.write(k["referenceTime"].encode('utf-8') + '\n\n')