Python 队列的使用

旧城等待， 2023-07-02 10:30 18阅读 0赞

### 代码如下，代码分析待续 ###

#coding:utf-8
    """ Author:fanbinglin Date: 2020/01/28 21点22分 功能: 下载网站小说并保存 http://www.xbiquge.la/5/5443/ """
    import requests
    from lxml import etree
    import re
    from bs4 import BeautifulSoup as bf
    import os
    import time
    from queue import Queue
    import threading
    #创建条件变量对象
    condition_html = threading.Condition()
    condition_save = threading.Condition()
    # os.system("del *.txt")
    class downLoad:
        def __init__(self):
            self.header = { 
            }
            self.url_queue = Queue()
            self.html_queue = Queue()
            self.save_queue = Queue()
            urlIndex = 'http://www.xbiquge.la/5/5443/'
            dns = 'http://www.xbiquge.la/'
            finishFlag = readFile('finishUrlList.txt')
            # print(finishFlag)
            finishUrlList = []
            if finishFlag:
                finishUrlList = [x for x in finishFlag.split('\n') if x != ""]
            self.finishUrlList = finishUrlList
            totleFlag = readFile('totle.txt')
            if totleFlag:
                totleList = [x for x in totleFlag.split('\n') if x != ""]
            else:
                response = requests.get(urlIndex)
                response.encoding = response.apparent_encoding
                html = bf(response.text,'html.parser')
                aTagList = html.select('dd a')
                totleList = [urlIndex + str(y) + '.html' for y in sorted(list(set([int(x['href'].split('/')[-1].split(r'.')[0]) for x in aTagList])))]
                saveFile('totle.txt',"\n".join(totleList),'w+')
            self.urlList = [x for x in totleList if x not in finishUrlList]
            print(len(totleList)-len(self.urlList))
            time.sleep(1)
        def get_totle_url(self):
            [self.url_queue.put(x) for x in self.urlList] 
        def parse_html(self):
            global condition_html
            while self.url_queue.not_empty:
                condition_html.acquire()
                url = self.url_queue.get()
                response = requests.get(url)
                response.encoding = response.apparent_encoding
                html = bf(response.text,'html.parser')
                html.url = url
                self.html_queue.put(html)
                self.url_queue.task_done()
                condition_html.release()
        def save_content(self):
            global condition_save
            while self.html_queue.not_empty:
                condition_save.acquire()
                html = self.html_queue.get()
                try:
                    content = html.select_one('#content').text.replace(' ','') + '\n'
                    url = html.url
                    print(url)
                    indexId = url.split(r'/')[-1]
                    with open(indexId + '.txt','w+',encoding = 'utf-8') as f:
                        f.write(content)
                        f.close()
                        saveFile('finishUrlList.txt',url + '\n','a+')
                except:
                    print("ERROR")
                self.html_queue.task_done()
                condition_save.release()
        def run(self):
            thread_list = []
            thread_url = threading.Thread(target=self.get_totle_url)
            thread_list.append(thread_url)
            for i in range(4):
                thread_parse = threading.Thread(target=self.parse_html)
                thread_list.append(thread_parse)
            for k in range(15):
                thread_save = threading.Thread(target=self.save_content)
                thread_list.append(thread_save)
            thread_list[0].setDaemon(True)
            thread_list[0].start()
            time.sleep(2)
            for t in thread_list[1:]:
                t.setDaemon(True)
                t.start()
                print(t.name)
            self.url_queue.join()
            self.html_queue.join()
            self.save_queue.join()
    
    def saveFile(filename,content,mode):
        try:
            #with open(filename,mode,encoding='utf-8') as f:#二进制文件不能保存为utf-8格式
            with open(filename,mode,encoding='utf-8') as f:
                f.write(content)
                f.close()
                print('\t'+filename + "保存成功!")
        except:
            print('\t'+filename + "保存失败!")
            f.close()
            
    def readFile(filename):
        try:
            with open(filename,"r",encoding='utf-8') as f:
                return f.read()
        except:
            return False
    if __name__ == "__main__":
        mode = 0
        if mode == 1:
            txtList = [x for x in os.listdir() if x[0:6].isdigit()]
            saveFile('panlong.txt',''.join([readFile(x) for x in txtList]),'w')
        else:
            startTime = time.time()
            story = downLoad()
            story.run()
            endTime = time.time()
            print(endTime-startTime)

acquire(\*args)  
请求底层锁。此方法调用底层锁的相应方法，返回值是底层锁相应方法的返回值。

release()  
释放底层锁。此方法调用底层锁的相应方法。没有返回值。