文本分类-朴素贝叶斯分类器

「爱情、让人受尽委屈。」 2022-06-03 03:54 262阅读 0赞

判断输入句子向量是侮辱性语言还是正常言论。

from numpy import *
    
    def loadDataSet():
        postingList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
                       ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
                       ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
                       ['stop', 'posting', 'stupid', 'wprthless', 'garbage'],
                       ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
                       ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']] #进行词条切分后的文档集合
        classVec = [0, 1, 0, 1, 0, 1] #1表示侮辱性文字，0表示正常言论
        return postingList, classVec
    
    def createVocabList(dataSet):
        vocabSet = set([]) #创建一个空子集
        for document in dataSet:
            vocabSet = vocabSet | set(document)  #set子集并运算，创建一个不重复词列表
        return list(vocabSet)
    
    #检查某个词是否在vocabList中
    def setOfWords2Vec(vocabList, inputSet): #vocabList词汇表，inputSet输入文档
        returnVec = [0]*len(vocabList) #创建一个与vocabList等长的0向量
        for word in inputSet:
            if word in vocabList:
                returnVec[vocabList.index(word)] = 1 #相应的位置置1
            else: print("the word: %s is not in my Vocabulary!" % word)
        return returnVec
    
    def trainNB0(trainMatrix, trainCategory): #trainMatrix为判断矩阵，trainCategory为文档标签
        numTrainDocs = len(trainMatrix) #判断列表长度
        numWords = len(trainMatrix[0]) #创建一个列表
        pAbusive = sum(trainCategory)/float(numTrainDocs) #侮辱性句子言论占总言论百分比
        # p0Num = zeros(numWords) #创建一个与numWords等长的零向量，正常言论的单词
        # p1Num = zeros(numWords) #侮辱性的单词
        p0Num = ones(numWords)
        p1Num = ones(numWords)
        # p0Denom = 0.0 #正常
        # p1Denom = 0.0 #侮辱性
        p0Denom = 2.0
        p1Denom = 2.0 #防止分母为0
        for i in range(numTrainDocs):
            if trainCategory[i] == 1: #若这一句表示含义为侮辱性
                p1Num += trainMatrix[i] #p1Num中是每个单词出现的数量，列表对应相加
                p1Denom += sum(trainMatrix[i]) #单词数
            else: #正常言论
                p0Num += trainMatrix[i]
                p0Denom += sum(trainMatrix[i])
        # p1Vect = p1Num/p1Denom  #每个侮辱单词在侮辱单词总数的百分比
        # p0Vect = p0Num/p0Denom
        p1Vect = log(p1Num / p1Denom) #采用log防止计算下溢出
        p0Vect = log(p0Num / p0Denom)
        return p0Vect, p1Vect, pAbusive
    
    def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1): #vec2Classify为要分类的向量，判断输入句子属于那种言论
        p1 = sum(vec2Classify * p1Vec) + log(pClass1)  #向量对应相乘，得到词汇是侮辱性的概率，然后所有概率log值相加，然后再乘以侮辱性的概率
        # p(c1|w)=p(w|c1)p(c1) 分母相同没有计算
        p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)
        if p1 > p0:
            return 1  #侮辱性言论
        else:
            return 0  #正常言论
    
    def testingNB():
        listOPosts, listClasses = loadDataSet() #创建文档集合以及标签
        myVocabList = createVocabList(listOPosts) #创建不重复的单词集
        trainMat = []
        for postinDoc in listOPosts:
            trainMat.append(setOfWords2Vec(myVocabList, postinDoc))  # trainMat中是每个文档是否在单词集中
        p0V, p1V, pAb = trainNB0(array(trainMat), array(listClasses)) #array(trainMat)二维矩阵，array(listClasses)一维矩阵
        testEntry = ['love', 'my', 'dalmation']
        thisDoc = array(setOfWords2Vec(myVocabList, testEntry)) #检查testEntry是否在vocabList中
        print(testEntry, "classified as:", classifyNB(thisDoc, p0V, p1V, pAb))
        testEntry = ['stupid', 'garbage']
        thisDoc = array(setOfWords2Vec(myVocabList, testEntry)) #检查testEntry是否在vocabList中
        print(testEntry, "classified as:", classifyNB(thisDoc, p0V, p1V, pAb))
    
    #测试
    testingNB()
    
    >>['love', 'my', 'dalmation'] classified as: 0
      ['stupid', 'garbage'] classified as: 1