爬取某一微博用户所有文本文件或者是视频图片文件

谁践踏了优雅 2022-05-14 15:06 136阅读 0赞

## 简述 ##

爬取所有的中大微博的信息。包括文本文件和视频文件，亦或是是图片文件。

## 代码实现 ##

from gevent import monkey
    import gevent
    
    monkey.patch_all(select=False)
    import selenium
    from selenium import webdriver
    import time
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    import os
    import shutil
    import requests
    
    
    def scroll():
        try:
            browser.find_element_by_xpath('//div[@class="W_pages"]')
        except:
            return False
        return True
    
    
    def checkfile(filepath):
        if os.path.exists(filepath):
            shutil.rmtree(filepath)
        os.mkdir(filepath)
    
    
    def writeText(text, filepath):
        with open(os.path.join(filepath, 'text.txt'), 'w', encoding='utf-8', errors='ignore') as f:
            f.write(text)
    
    
    def writeMedia(mediaurl, filepath, name, vedio=False):
        if not vedio:
            form = os.path.splitext(mediaurl)[-1]
        else:
            form = '.mp4'
        with open(os.path.join(filepath, name + form), 'wb') as f:
            f.write(requests.get(mediaurl).content)
    
    
    if __name__ == '__main__':
        options = webdriver.ChromeOptions()
        options.binary_location = r"D:\Software\Chrome\Application\chrome.exe"
        browser = webdriver.Chrome(chrome_options=options)
        url = 'https://weibo.com/u/1892723783?is_all=1'
        browser.get(url)
        time.sleep(40)
        WB_innerwrap = browser.find_element_by_css_selector('div[class="WB_innerwrap"]')
        # 总共的数量
        td_last_number = int(WB_innerwrap.find_elements_by_tag_name('td')[-1].find_element_by_class_name("W_f16").text)
        print("微博总数:", td_last_number)
    
        file_path = './weibo'
    
        checkfile(file_path)
        medialist = []
        MAXLEN = 50
        for TIMES in range(221):
            while not scroll():
                browser.execute_script("window.scrollBy(0,10000)")
                time.sleep(5)
    
            WB_feed = browser.find_element_by_css_selector('div[module-type="feed"]')
            feed_content = WB_feed.find_elements_by_xpath('//div[@node-type="feed_content"]')
            for feed_list_item in feed_content:
                print("#### BEGIN", td_last_number)
                WB_detail = feed_list_item.find_element_by_class_name('WB_detail')
                WB_text = WB_detail.find_element_by_css_selector('div[node-type="feed_list_content"]').text
    
                # 创建文件夹
                feed_list_item_file_path = os.path.join(file_path, str(td_last_number))
                checkfile(feed_list_item_file_path)
    
                writeText(WB_text, feed_list_item_file_path)
    
                td_last_number -= 1
                try:
                    WB_media_box = WB_detail.find_element_by_class_name('media_box')
                    imgs = WB_media_box.find_elements_by_tag_name('img')
                    for i, img in enumerate(imgs):
                        # print('image ', i, img.get_attribute('src'))
                        medialist.append(gevent.spawn(writeMedia, img.get_attribute('src'), feed_list_item_file_path,
                                                      'image ' + str(i)))
                    videos = WB_media_box.find_elements_by_tag_name('video')
                    for video in videos:
                        # print('video', video.get_attribute('src'))
                        medialist.append(gevent.spawn(writeMedia, video.get_attribute('src'), feed_list_item_file_path,
                                                      'veido', True))
                except:
                    pass
                if len(medialist) >= MAXLEN:
                    gevent.joinall(medialist)
                    medialist = []
    
            W_pages = browser.find_element_by_xpath('//div[@class="W_pages"]')
            W_pages.find_elements_by_tag_name('a')[-1].click()
    
            if len(medialist):
                gevent.joinall(medialist)
                medialist = []
            else:
                time.sleep(5)