Python，网络爬虫爬取百度贴吧元数据绘制matplotlib图

柔情只为你懂 2022-12-30 10:38 188阅读 0赞

import requests
    import re
    import time
    import pandas as pd
    from tqdm import tqdm
    import matplotlib
    import matplotlib.pyplot as plt
    
    
    def get_city_data(city):
        url = f'https://tieba.baidu.com/f?ie=utf-8&kw={city}'
        r = requests.get(url=url, timeout=30)
    
        card_menNum_pattern = re.compile(r'<span class="card_menNum">(.*?)</span>', re.S)
        card_menNum_result = re.search(card_menNum_pattern, r.text)
        card_menNum = card_menNum_result.groups()[0]
        card_menNum = card_menNum.replace(',', '')
    
        card_infoNum_pattern = re.compile(r'<span class="card_infoNum">(.*?)</span>', re.S)
        card_infoNum_result = re.search(card_infoNum_pattern, r.text)
        card_infoNum = card_infoNum_result.groups()[0]
        card_infoNum = card_infoNum.replace(',', '')
    
        return city, card_menNum, card_infoNum, time.strftime('%Y-%m-%d', time.localtime())
    
    
    def drawchart(df):
        myfont = matplotlib.font_manager.FontProperties(fname='C:\Windows\Fonts\msyh.ttc')
        plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号
        plt.rc('font', family='YaHei', weight='bold')
    
        order = []
        name = []
        mem = []
        paper = []
        str_time = []
        for d, i in zip(df.values, df.index):
            order.append(i)
            name.append(d[0])
            mem.append(int(d[1]))
            paper.append(d[2])
            str_time.append(d[3])
    
        FONT_SIZE = 18
    
        fig, ax = plt.subplots(figsize=(15, 13))
    
        b = ax.barh(y=range(len(name)), width=mem, align='center', color='red')
    
        # 为横向水平的柱图右侧添加数据标签。
        i = 0
        for rect in b:
            w = rect.get_width()
            ax.text(x=w, y=rect.get_y() + rect.get_height() / 2, s='%d' % (int(w)),
                    horizontalalignment='left', verticalalignment='center',
                    fontproperties=myfont, fontsize=FONT_SIZE-1, color='green')
            ax.text(x=w / 2, y=rect.get_y() + rect.get_height() / 2, s=str(order[i]),
                    horizontalalignment='center', verticalalignment='center',
                    fontproperties=myfont, fontsize=FONT_SIZE-2, color='white')
            i = i + 1
    
        ax.set_yticks(range(len(name)))
        ax.set_yticklabels(name, fontsize=FONT_SIZE, fontproperties=myfont)
    
        ax.invert_yaxis()
    
        ax.set_xlabel('数据来源:百度贴吧', fontsize=FONT_SIZE + 5, fontproperties=myfont)
        ax.set_title('中国城市关注热度排名 ({when})'.format(when=str_time[0]), fontsize=FONT_SIZE + 10, fontproperties=myfont)
    
        # 不要横坐标上的label标签。
        plt.xticks(())
    
        # 清除四周的边框线
        ax.get_yaxis().set_visible(True)
        for spine in ["left", "top", "right", "bottom"]:
            ax.spines[spine].set_visible(False)
    
        plt.subplots_adjust(left=0.15)  # 调整左侧边距
    
        # ax.margins(y=0.01) #缩放 zoom in
    
        ax.set_aspect('auto')
    
        plt.show()
    
    
    def main():
        city = ['北京', '上海', '广州', '深圳', '成都',
                '天津', '南京', '杭州', '重庆', '武汉',
                '长沙', '厦门', '郑州', '沈阳', '西安',
                '大连', '济南', '青岛', '苏州', '昆明',
                '合肥', '福州', '哈尔滨', '太原', '宁波',
                '贵阳', '南宁', '长春', '南昌', '石家庄', '东莞',
                ]
    
        city_data = []
        pbar = tqdm(total=len(city), leave=True)
        for c in city:
            result = get_city_data(c)
            city_data.append(list(result))
    
            pbar.update(1)
            time.sleep(3)
    
        col = ['城市', '关注', '发帖', '时间']
        df = pd.DataFrame(data=city_data, columns=col)
        df = df.sort_values(by=col[1], axis=0, ascending=False)  # 降序
    
        # 排序后重置index，
        # 否则索引是混乱的
        df = df.reset_index(drop=True)
    
        # 因为默认的pandas起始索引从0开始，
        # 为了使数据行的初始索引（起始索引index）从1开始
        df.index = df.index + 1
    
        print(df.head(10))
    
        drawchart(df)
    
    
    if __name__ == '__main__':
        main()

输出：

100%|██████████| 31/31 [02:03<00:00,  4.03s/it]    
    城市       关注        发帖          时间
    1   成都  3306806  93030375  2020-12-24
    2   深圳  2995475  67354620  2020-12-24
    3   北京  2665417  43816255  2020-12-24
    4   广州  2658129  58594452  2020-12-24
    5   郑州  2577555  93304019  2020-12-24
    6   重庆  2383661  63218055  2020-12-24
    7   上海  2147059  15954572  2020-12-24
    8   西安  2009700  81571724  2020-12-24
    9   杭州  2001891  24195340  2020-12-24
    10  武汉  2000678  40920811  2020-12-24

图：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3poYW5ncGhpbA_size_16_color_FFFFFF_t_70][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3poYW5ncGhpbA_size_16_color_FFFFFF_t_70]: /images/20221120/c682758347674efe9ed78e4b837d06b9.png