爬取百度贴吧所有精品贴照片

谁借莪１个温暖的怀抱￠ 2022-06-05 03:56 272阅读 0赞

#coding=utf-8
    import os
    import random
    import requests
    from lxml import etree
    from urllib.parse import urlparse
    import urllib.request as urllib
    from bs4 import BeautifulSoup
    
    
    user_agent_list = ["Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 ",  
                "(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",    
                "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11",   
                "(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",    
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6",   
                "(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",    
                "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6",  
                "(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",    
                "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 "    
                "(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",    
                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5",   
                "(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",    
                "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5",   
                "(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",    
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "    
                "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",    
                "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3",    
                "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",    
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 "    
                "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",    
                "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3",   
                "(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",    
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3",   
                "(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",    
                "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3",  
                "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",    
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3",    
                "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",    
                "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3",   
                "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",    
                "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3",   
                "(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",    
                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24",    
                "(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",    
                "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 ",   
                "(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",] 
    
    
    
    class Crawler(object):
        def __init__(self,start_url):
            self.index = 1
            self.tag = 0
            self.tagname = []
            self.start_url = start_url
            self.domain = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(self.start_url))
            
        @staticmethod
        def request(url, **kwargs):
            try:
                page = requests.get(url,**kwargs)
                return page.text
            except:
                return ''
            
        def get_max_page(self,html,regular):
            html = etree.HTML(html)
            pages = html.xpath(regular)
            try:
                max_page,*_ = pages
            except:
                max_page = '1'
            return max_page
    
        def get_good_pages(self,html):
            '''
            获取所有精品帖子的页面，类似
            https://tieba.baidu.com/f?kw=图片&ie=utf-8&tab=good&cid=&pn=50
            
            '''
            max_page = self.get_max_page(html,'//a[@class="last"]//@href')
            if max_page != '1':
                max_page = max_page.split('=')[-1]
            max_page = int(max_page) + 1
            for i in range(0,max_page,50):
                yield (self.start_url.split('&')[0] + '&ie=utf-8&cid=0&pn={}'.format(i))
                
    
        def get_good_urls(self,html):
            '''
            获取精品帖子里面内的所有精品贴url，类似
            https://tieba.baidu.com/p/3868212854
            '''
            pages = self.get_good_pages(html)
            for page in pages:
                html = self.request(page)
                html = etree.HTML(html)
                self.tagname.extend(html.xpath('//a[@class = "j_th_tit"]//@title'))
                urls = html.xpath('//a[@class = "j_th_tit"]//@href')
                for url in urls:
                    url = url.split('?')[0]
                    url = self.domain + url
                    yield url
                    self.tag += 1
                    self.index = 1
                    
        def get_post_urls(self,url,max_page):
            '''
            获取所有精品帖子的url，类似
            https://tieba.baidu.com/p/3868212854?pn=2
            '''
            for i in range(1,max_page + 1):
                yield (url + '?pn={}'.format(i))
    
        def get_single_urls(self,html):
            urls = self.get_good_urls(html)
            for url in urls:
                html = self.request(url)
                max_page = self.get_max_page(html,'//li[@class = "l_reply_num"]//@max-page')
                max_page = int(max_page) + 1
                single_urls = self.get_post_urls(url,max_page)
                yield single_urls
    
        def get_imgs(self,html):
            '''
           下载某个精品帖子的所有图片
           '''
            jpgdir = r'D:\pic\{}'.format( self.tagname[self.tag])
            if not os.path.exists(jpgdir):
                os.makedirs(jpgdir)
    
            html = etree.HTML(html)
            img_urls = html.xpath('//img[@class = "BDE_Image" and @width > "400"]//@src')
            for img in img_urls:
                print ("正在下载第{}张图片".format(self.index))
                urllib.urlretrieve(img,r'{}\{}.jpg'.format(jpgdir,self.index))
                self.index += 1
    
             
        def run(self,html):
            single_urls = self.get_single_urls(html)
            for single_url in single_urls:
                for url in single_url:
                    User_Agent = random.choice(user_agent_list) #伪装一下
                    headers = {'User-Agent':User_Agent}  
                    html = self.request(url,headers = headers)
                    self.get_imgs(html)
               
    if __name__=='__main__':
        post_bar = input("请输入贴吧名称:")
        start_url = 'https://tieba.baidu.com/f/good?kw={}&ie=utf-8&cid=0&pn=0'.format(post_bar)
        crawler = Crawler(start_url)
        html = crawler.request(start_url)
        if '本吧暂不开放' in html:
            print ("抱歉，根据相关法律法规和政策，本吧暂不开放。")
        elif 'page404' in html:
            print ('很抱歉，您要访问的页面不存在。')
        else:
            print("开始爬取{}吧所有精品帖子图片".format(post_bar))
            crawler.run(html)