【爬虫】多线程、多进程方式爬取糗事百科段子（xpath、etree）

Love The Way You Lie 2022-11-22 00:12 223阅读 0赞

#### 1.简单版 ####

因为我们要爬取的网页结构比较简单，这里就不分析了，直接上代码

import time
    import requests
    from lxml import etree
    
    
    class QiuShiSpider(object):
        def __init__(self):
            self.base_url = "https://www.qiushibaike.com"
            self.headers = { 
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
            }
            self.start_page = 1
            self.end_page = 5
           
        def get_html(self, page):
            res = requests.get(url=self.base_url + '/text/page/' + str(page), headers=self.headers)
            if res.status_code == 200:
                return res.content.decode()
    
        def get_content(self, html):
            html = etree.HTML(html)
            contents_els = html.xpath('//div[@class="content"]')
            contents_list = []
            for i in range(len(contents_els)):
                content = contents_els[i].xpath('string(.)')
                if "查看全文" in content:
                    html = requests.get(self.base_url + contents_els[i].xpath('../@href')[0])
                    html = etree.HTML(html.content.decode())
                    try:
                        print(etree.tostring(html, encoding="utf-8").decode())
                        content = html.xpath('//div[@class="content"]')[0].xpath('string(.)')
                    except Exception as e:
                        print(e)
                contents_list.append(content.replace("\n\n", ""))
            # print(contents_list)
            return contents_list
    
        @staticmethod
        def save_data(filename, data_list):
            with open(filename, "w", encoding="utf-8") as f:
                for d in data_list:
                    f.write(d + "\n\n")
    
        def run(self):
            for i in range(self.start_page, self.end_page):
                print(f'正在爬取第{i}页')
                html = self.get_html(i)
                contents_list = self.get_content(html)
                self.save_data("糗事" + str(i) + '.txt', contents_list)
    
    
    if __name__ == '__main__':
        t1 = time.time()
        spider = QiuShiSpider()
        spider.run()
        t2 = time.time()
        print("耗时：", t2 - t1)

#### 2.多线程版 ####

知识点回顾

# 创建线程
    from threading import Thread
    t1 = threading.Thread(target=func,args=(,))  # 创建线程多对象
    t1.setDaemon(True)  # 守护多线程
    t1.start()  # 启动线程
    
    # 创建队列
    from queue import Queue
    q = Queue(maxsize=100)
    item = { }
    q.put(item)  # 把数据放入到队列中，如果队列数据已满，就不进行等待，引发异常
    q.put_nowait(item)  # 把数据从队列中取出，如果队列为空，就进行等待直到队列不为空，返回数据
    q.get()  # 把数据从队列中取出，如果队列为空，就不进行等待，引发异常
    q.get_nowait()
    q.qsize()  # 获取队列中元素个数
    q.unfinished_tasks  # 获取未完成的任务数
    q.task_done()  # 让 unfinished_tasks 属性 -1
    q.join()  # 当前程序进行等待状态，直到队列的 unfinished_tasks 属性为 0 时继续执行

import time
    import requests
    from lxml import etree
    from queue import Queue
    from threading import Thread
    
    
    class QiuShiSpider(object):
        def __init__(self):
            self.base_url = "https://www.qiushibaike.com"
            self.headers = { 
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
            }
            self.start_page = 1
            self.end_page = 11
    
            self.url_queue = Queue()
            self.html_queue = Queue()
            self.data_queue = Queue()
    
        def get_urls(self):
            for i in range(self.start_page, self.end_page):
                url = self.base_url + '/text/page/' + str(i)
                self.url_queue.put(url)  # 将url添加到队列
    
        def get_html(self):
            while True:
                url = self.url_queue.get()
                res = requests.get(url=url, headers=self.headers)
                if res.status_code == 200:
                    # return res.content.decode()
                    self.html_queue.put(res.content.decode())  # 将数据存入html队列容器中
                    self.url_queue.task_done()  # 通知url队列任务完成，任务数-1
    
        def get_content(self):
            while True:
                html = self.html_queue.get()  # 从html队列中取出html数
                html = etree.HTML(html)
                contents_els = html.xpath('//div[@class="content"]')
                contents_list = []
                for i in range(len(contents_els)):
                    content = contents_els[i].xpath('string(.)')
                    if "查看全文" in content:
                        html = requests.get(self.base_url + contents_els[i].xpath('../@href')[0])
                        html = etree.HTML(html.content.decode())
                        try:
                            content = html.xpath('//div[@class="content"]')[0].xpath('string(.)')
                        except Exception as e:
                            print(i)
                            print(e)
                    contents_list.append(content.replace("\n\n", ""))
                # print(contents_list)
                # return contents_list
                self.data_queue.put(contents_list)
                self.html_queue.task_done()
    
        def save_data(self):
            while True:
                data_list = self.data_queue.get()
                with open("糗事" + str(time.time()) + ".txt", "w", encoding="utf-8") as f:
                    for d in data_list:
                        f.write(d + "\n\n")
                self.data_queue.task_done()
    
        def run(self):
            thread_list = []
            url_thread = Thread(target=self.get_urls)
            html_thread = Thread(target=self.get_html)
            content_thread = Thread(target=self.get_content)
            save_thread = Thread(target=self.save_data)
            thread_list.append(url_thread)
            thread_list.append(html_thread)
            thread_list.append(content_thread)
            thread_list.append(save_thread)
    
            for thread in thread_list:
                thread.setDaemon(True)
                thread.start()
    
            for queue in [self.url_queue, self.html_queue, self.data_queue]:
                queue.join()
    
    
    if __name__ == '__main__':
        t1 = time.time()
        spider = QiuShiSpider()
        spider.run()
        t2 = time.time()
        print(t2 - t1)

#### 3.多进程版 ####

多进程跟多线程没有太大区别

import time
    import requests
    from lxml import etree
    # from queue import Queue
    from multiprocessing import JoinableQueue as Queue
    # from threading import Thread
    from multiprocessing import Process
    
    
    class QiuShiSpider(object):
        def __init__(self):
            self.base_url = "https://www.qiushibaike.com"
            self.headers = { 
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
            }
            self.start_page = 1
            self.end_page = 11
    
            self.url_queue = Queue()
            self.html_queue = Queue()
            self.data_queue = Queue()
    
        def get_urls(self):
            for i in range(self.start_page, self.end_page):
                url = self.base_url + '/text/page/' + str(i)
                self.url_queue.put(url)  # 将url添加到队列
    
        def get_html(self):
            while True:
                url = self.url_queue.get()
                res = requests.get(url=url, headers=self.headers)
                if res.status_code == 200:
                    # return res.content.decode()
                    self.html_queue.put(res.content.decode())  # 将数据存入html队列容器中
                    self.url_queue.task_done()  # 通知url队列任务完成，任务数-1
    
        def get_content(self):
            while True:
                html = self.html_queue.get()  # 从html队列中取出html数
                html = etree.HTML(html)
                contents_els = html.xpath('//div[@class="content"]')
                contents_list = []
                for i in range(len(contents_els)):
                    content = contents_els[i].xpath('string(.)')
                    if "查看全文" in content:
                        html = requests.get(self.base_url + contents_els[i].xpath('../@href')[0])
                        html = etree.HTML(html.content.decode())
                        try:
                            content = html.xpath('//div[@class="content"]')[0].xpath('string(.)')
                        except Exception as e:
                            print(i)
                            print(e)
                    contents_list.append(content.replace("\n\n", ""))
                # print(contents_list)
                # return contents_list
                self.data_queue.put(contents_list)
                self.html_queue.task_done()
    
        def save_data(self):
            while True:
                data_list = self.data_queue.get()
                with open("糗事" + str(time.time()) + ".txt", "w", encoding="utf-8") as f:
                    for d in data_list:
                        f.write(d + "\n\n")
                self.data_queue.task_done()
    
        def run(self):
            process_list = []
            url_thread = Process(target=self.get_urls)
            html_thread = Process(target=self.get_html)
            content_thread = Process(target=self.get_content)
            save_thread = Process(target=self.save_data)
            process_list.append(url_thread)
            process_list.append(html_thread)
            process_list.append(content_thread)
            process_list.append(save_thread)
    
            for process in process_list:
                # process.daemon = True
                process.start()
    
            for queue in [self.url_queue, self.html_queue, self.data_queue]:
                queue.join()
    
    
    if __name__ == '__main__':
        t1 = time.time()
        spider = QiuShiSpider()
        spider.run()
        t2 = time.time()
        print(t2 - t1)