python统计搜狗实验室新闻词频绘制词云

梦里梦外; 2023-06-07 08:08 11阅读 0赞

## 1.下载数据 ##

数据下载地址 https://www.sogou.com/labs/resource/ca.php  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3JqMjAxNzIxMTgxMQ_size_16_color_FFFFFF_t_70]  
下载后解压数据，发现是.dat格式的文件，  
用python打开：  
![在这里插入图片描述][20191013190501711.PNG]

## 2.分词 ##

该.dat文件格式为gbk，首先应读取该文件内容，然后将文件内容转换为utf-8  
由于文件太大，所以不能一次性读取，一次性读取数据会一次性全部加载到内存，电脑会立马卡死，应该分行读取

for line in fp:
            str1 = line.decode('gbk', 'ignore')

我们要做的就是匹配<content> 和<contenttitle> 的内容

i = 1
      for line in fp:
            str1 = line.decode('gbk', 'ignore')
             if i%6==4:
                print('contenttitle:')
             	re.findall('([^<contenttitle>(*?)</contenttitle>（）', line)
             if i%6==5:
             	print('content:')
             	re.findall('([^<content>(*?)</content>（）', line)

这样就能获得匹配content和contenttitle的内容了  
当然我们还应该对其中内容进行详细的匹配规则，去除一些字符  
如下

'([^<contenttitle>(*?)</contenttitle>（）？0１２３４５６７８９０ＡＧ＊·《》！】Ｃ＃：“”、。，ＩＶ\u3000$]+).'

由于数据量太大，我们为content和contenttitle分别开一个线程  
首先写一个对内容进行处理的函数

def processContent(list,str,pattern):
        import re#正则表达式库
        import jieba#结巴库
        contentList = re.findall(pattern, str)
        if len(contentList) > 0:
            # print('contenttitle',i,':',list1[0])
            for j in jieba.lcut(contentList[0],cut_all=False):
                list.append(j)
            print('list:', list)

在主函数中调用该函数，在分别使用两个线程调用该函数

import threading#线程模块
        ##按行读取文件,获取content和contenttitle内容
        fp = open('news_tensite_xml.dat','rb')
        i = 1
        list1 = []
        for line in fp:
            str1 = line.decode('gbk', 'ignore')
            t1=threading.Thread(target=countContent,args=(list1,str1,'([^<contenttitle>(*?)</contenttitle>（）？0１２３４５６７８９０ＡＧ＊·《》！】Ｃ＃：“”、。，ＩＶ\u3000$]+).'))
            t2=threading.Thread(target=countContent,args=(list1,str1,'([^<content>(*?)</content>（）？0１２３４５６７８９０ＡＧ·＊《》！】＃：“”、。，ＩＶ\u3000$]+).'))
            if i%6==4:
                print('contenttitle:')
                ##countContent(list1,str1,'([^<contenttitle>(*?)</contenttitle>$]+).')
                t1.start()
                t1.join()
            if i%6==5:
                print('content:')
                ##countContent(list1, str1, '([^<content>(*?)</content>$]+).')
                t2.start()
                t2.join()
            i = i + 1

## 3.统计词频 ##

使用collections的Counter函数可用将一个列表转换为一个集合，并统计每个词语的频率

for i in list1:
            if i not in remove_words:
                words.append(i)
        word_counts = collections.Counter(words)
        print('word_counts:',word_counts)
        words_common = word_counts.most_common(100)
        print('top_100:',words_common)

## 4.绘制词云 ##

安装wordcloud词云展示库，PIL图像处理库，numpy数值处理库，matplotlib.pyplot图像展示库

import wordcloud#词云展示库
        from PIL import Image#图像处理库
        import numpy#数值处理库
        import matplotlib.pyplot as plt#图像展示库
        mask =numpy.array(Image.open('china.jpg'))
        worldCloud = wordcloud.WordCloud(background_color='white',min_font_size=30,max_font_size=100,width=400,mask=mask,font_path="simkai.ttf")
        worldCloud.generate_from_frequencies(words)
        #输出到文件
        worldCloud.to_file('wordCloud.jpg')
        #image_colors = wordcloud.ImageColorGenerator(mask)  # 从背景图建立颜色方案
        #worldCloud.recolor(color_func=image_colors)  # 将词云颜色设置为背景图方案
        plt.imshow(worldCloud)
        plt.axis("off")
        plt.show()

完整代码：  
countWords.py

#绘制词云
    def produceWordCloud(words):
        import wordcloud#词云展示库
        from PIL import Image#图像处理库
        import numpy#数值处理库
        import matplotlib.pyplot as plt#图像展示库
        mask =numpy.array(Image.open('china.jpg'))
        worldCloud = wordcloud.WordCloud(background_color='white',min_font_size=5,max_font_size=50,width=1000,height=800,mask=mask,max_words=100,font_path="simkai.ttf")
        worldCloud.generate_from_frequencies(words)
        #输出到文件
        worldCloud.to_file('keywords.jpg')
        #image_colors = wordcloud.ImageColorGenerator(mask)  # 从背景图建立颜色方案
        #worldCloud.recolor(color_func=image_colors)  # 将词云颜色设置为背景图方案
        plt.imshow(worldCloud)
        plt.axis("off")
        plt.show()
    #分词
    def divideWords(list,str,pattern):
        import re#正则表达式库
        import jieba#结巴库
        print('正在分词...',end=' ')
        contentList = re.findall(pattern, str)
        if len(contentList) > 0:
            # print('contenttitle',i,':',list1[0])
            for j in jieba.lcut(contentList[0],cut_all=False):
                list.append(j)
            #print('list:', list)
        print('分词完成...',end=' ')
    #统计词频
    def countWordFrequentcy(list,word_counts,remove_words):
        import time
        print(time.ctime(),'正在统计词频...',end='\n')
        words=[]
        import collections  # 集合模块
        for i in list:#去除一些词语
            if len(i)>=2:#i not in remove_words and
                words.append(i)
        word_counts = collections.Counter(words)
        print('word_counts:', word_counts)
        words_common = word_counts.most_common(100)
        print('top_100:', words_common)
        print(time.ctime(),'词频统计完成...',end='\n')
        return word_counts
    
    
    def countWords():
        import time
        import threading  # 线程模块
        ##按行读取文件,获取content和contenttitle内
        #fp = open('news_tensite_xml.dat', 'rb')
        i = 1
        fileLength = len(["" for line in open('news_tensite_xml.dat', 'rb')])
        print('您将要读取',fileLength,'行数据......')
        time.sleep(10)
        list1 = []
        with open('news_tensite_xml.dat', 'rb') as fp:
            for line in fp:
                print(time.ctime(), '正在读取第', i, '行', end=' ')
                str1 = line.decode('gbk', 'ignore')
                t1 = threading.Thread(target=divideWords, args=(
                    list1, str1, '([^<contenttitle>(*?)</contenttitle>（）？0１２３４５６７８９０ＡＧＰＭａ＊＂．·《》／/！－【】ＹＣＬＢＪＳＴＮ＃ｈｅ：“”、。，ＩＶ\u3000$]+).'))
                t2 = threading.Thread(target=divideWords, args=(
                    list1, str1, '([^<content>(*?)</content>（）？0１２３４５６７８９０ＡＧＰＭａ．·＊＂《》／/！－【】ＹＣＬＢＪＳＴＮｈｅ＃：“”、。，ＩＶ\u3000$]+).'))
                if i % 6 == 4:
                    print('读取到contenttitle', end=' ')
                    # divideWords(list1,str1,'([^<contenttitle>(*?)</contenttitle>$]+).')
                    t1.start()
                    t1.join()
                elif i % 6 == 5:
                    print('读取到content', end=' ')
                    # divideWords(list1, str1, '([^<content>(*?)</content>$]+).')
                    t2.start()
                    t2.join()
                else:
                    print('未读取到数据', end=' ')
                print('第', i, '行读取完成')
                #fileLength = 1000000
                if i <= fileLength :
                    i = i + 1
                    continue;
                else:
                    print(fileLength,'以及全部读完')
                    break;
    
    
        #print('list1:', list1)
        remove_words = ['临', '乎', '与', '为', '共', '冲', '到', '从', '即', '以', '于', '兜',
                        '似', '假', '去', '让', '诸', '及', '往', '迆', '连', '迎', '道', '遵', '对',
                        '导', '寻', '将', '当', '叫', '吃', '合', '向', '如', '尽', '问',
                        '打', '执', '把', '投', '拦', '按', '捉',
                        '洎', '给', '维', '缘', '在', '因', '惟', '就', '比',
                        '照', '较', '方', '拿', '替', '望', '朝',
                        '爰', '直', '由', '率', '被', '用', '至',
                        '管', '自', '起', '趁', '践', '跟', '仅', '的','是','当','被','一',
                        '了','和','新','年','月','日','你','都','后','最','有',
                        '称','大','据','近日','不','遭','不','下','上','前','曝','时','做','又']
        word_counts=[""]
        word_counts=countWordFrequentcy(list1,word_counts,remove_words)#统计词频
        produceWordCloud(word_counts)#分词
    
    def main():
    
        import  time
        start_time = time.ctime()
    
        countWords()
        end_time = time.ctime()
        print('start_time:',start_time)
        print('end_time:',end_time)
    
    if __name__ =='__main__':
        main()

打开命令行cmd，切换到该项目目录，输入python ./countWords.py程序就跑起来了  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3JqMjAxNzIxMTgxMQ_size_16_color_FFFFFF_t_70 1]

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3JqMjAxNzIxMTgxMQ_size_16_color_FFFFFF_t_70 2]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3JqMjAxNzIxMTgxMQ_size_16_color_FFFFFF_t_70]: /images/20230601/efb1bcd148c44060b9f3c90ddaabc094.png
[20191013190501711.PNG]: /images/20230601/d9d2dd147f6a480baf61c6dec557fa13.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3JqMjAxNzIxMTgxMQ_size_16_color_FFFFFF_t_70 1]: /images/20230601/9bca11d9dace40a8b19b873c78ffe803.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3JqMjAxNzIxMTgxMQ_size_16_color_FFFFFF_t_70 2]: /images/20230601/beb3d13dac2d4e53bb73ec3a4a3b7ae4.png