Hanlp之文本分类

柔情只为你懂 2022-12-15 12:42 285阅读 0赞

# 1、语料库格式 #

分类语料的根目录.目录必须满足如下结构: 
 根目录 
 ├── 分类A 
 │ └── 1.txt 
 │ └── 2.txt 
 │ └── 3.txt 
 ├── 分类B 
 │ └── 1.txt 
 │ └── ... 
 └── ... 
 文件不一定需要用数字命名,也不需要以txt作为后缀名,但一定需要是文本文件.

![20201016165017777.png][]

# 2、项目格式 #

训练分类语料库要放到data/test/ 目录下

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2h5al9raW5n_size_16_color_FFFFFF_t_70][]

# 3、代码 #

## (1)TestUtility类 ##

/*
 * <author>Han He</author>
 * <email>me@hankcs.com</email>
 * <create-date>2018-06-23 11:05 PM</create-date>
 *
 * <copyright file="TestUtility.java">
 * Copyright (c) 2018, Han He. All Rights Reserved, http://www.hankcs.com/
 * This source is subject to Han He. Please contact Han He for more information.
 * </copyright>
 */
 package com.cn.test.TextClassification;
 
 import com.hankcs.hanlp.HanLP;
 
 import java.io.*;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
 /**
 * @author hankcs
 */
 public class TestUtility
 {
 static
 {
 ensureFullData();
 }
 
 public static void ensureFullData()
 {
 ensureData(HanLP.Config.PerceptronCWSModelPath, "http://nlp.hankcs.com/download.php?file=data", HanLP.Config.PerceptronCWSModelPath.split("data")[0], false);
 }
 
 /**
 * 保证 name 存在，不存在时自动下载解压
 *
 * @param name 路径
 * @param url 下载地址
 * @return name的绝对路径
 */
 public static String ensureData(String name, String url)
 {
 return ensureData(name, url, null, true);
 }
 
 /**
 * 保证 name 存在，不存在时自动下载解压
 *
 * @param name 路径
 * @param url 下载地址
 * @return name的绝对路径
 */
 public static String ensureData(String name, String url, String parentPath, boolean overwrite)
 {
 File target = new File(name);
 if (target.exists()) return target.getAbsolutePath();
 try
 {
 File parentFile = parentPath == null ? new File(name).getParentFile() : new File(parentPath);
 if (!parentFile.exists()) parentFile.mkdirs();
 String filePath = downloadFile(url, parentFile.getAbsolutePath());
 if (filePath.endsWith(".zip"))
 {
 unzip(filePath, parentFile.getAbsolutePath(), overwrite);
 }
 return target.getAbsolutePath();
 }
 catch (Exception e)
 {
 System.err.printf("数据下载失败，请尝试手动下载 %s 到 %s 。原因如下：\n", url, target.getAbsolutePath());
 e.printStackTrace();
 System.exit(1);
 return null;
 }
 }
 
 /**
 * 保证 data/test/name 存在
 *
 * @param name
 * @param url
 * @return
 */
 public static String ensureTestData(String name, String url)
 {
 return ensureData(String.format("data/test/%s", name), url);
 }
 
 /**
 * Downloads a file from a URL
 *
 * @param fileURL HTTP URL of the file to be downloaded
 * @param savePath path of the directory to save the file
 * @throws IOException
 * @author www.codejava.net
 */
 public static String downloadFile(String fileURL, String savePath)
 throws IOException
 {
 System.err.printf("Downloading %s to %s\n", fileURL, savePath);
 HttpURLConnection httpConn = request(fileURL);
 while (httpConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM || httpConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP)
 {
 httpConn = request(httpConn.getHeaderField("Location"));
 }
 
 // always check HTTP response code first
 if (httpConn.getResponseCode() == HttpURLConnection.HTTP_OK)
 {
 String fileName = "";
 String disposition = httpConn.getHeaderField("Content-Disposition");
 String contentType = httpConn.getContentType();
 int contentLength = httpConn.getContentLength();
 
 if (disposition != null)
 {
 // extracts file name from header field
 int index = disposition.indexOf("filename=");
 if (index > 0)
 {
 fileName = disposition.substring(index + 10,
 disposition.length() - 1);
 }
 }
 else
 {
 // extracts file name from URL
 fileName = new File(httpConn.getURL().getPath()).getName();
 }
 
 // System.out.println("Content-Type = " + contentType);
 // System.out.println("Content-Disposition = " + disposition);
 // System.out.println("Content-Length = " + contentLength);
 // System.out.println("fileName = " + fileName);
 
 // opens input stream from the HTTP connection
 InputStream inputStream = httpConn.getInputStream();
 String saveFilePath = savePath;
 if (new File(savePath).isDirectory())
 saveFilePath = savePath + File.separator + fileName;
 String realPath;
 if (new File(saveFilePath).isFile())
 {
 System.err.printf("Use cached %s instead.\n", fileName);
 realPath = saveFilePath;
 }
 else
 {
 saveFilePath += ".downloading";
 
 // opens an output stream to save into file
 FileOutputStream outputStream = new FileOutputStream(saveFilePath);
 
 int bytesRead;
 byte[] buffer = new byte[4096];
 long start = System.currentTimeMillis();
 int progress_size = 0;
 while ((bytesRead = inputStream.read(buffer)) != -1)
 {
 outputStream.write(buffer, 0, bytesRead);
 long duration = (System.currentTimeMillis() - start) / 1000;
 duration = Math.max(duration, 1);
 progress_size += bytesRead;
 int speed = (int) (progress_size / (1024 * duration));
 float ratio = progress_size / (float) contentLength;
 float percent = ratio * 100;
 int eta = (int) (duration / ratio * (1 - ratio));
 int minutes = eta / 60;
 int seconds = eta % 60;
 
 System.err.printf("\r%.2f%%, %d MB, %d KB/s, ETA %d min %d s", percent, progress_size / (1024 * 1024), speed, minutes, seconds);
 }
 System.err.println();
 outputStream.close();
 realPath = saveFilePath.substring(0, saveFilePath.length() - ".downloading".length());
 if (!new File(saveFilePath).renameTo(new File(realPath)))
 throw new IOException("Failed to move file");
 }
 inputStream.close();
 httpConn.disconnect();
 
 return realPath;
 }
 else
 {
 httpConn.disconnect();
 throw new IOException("No file to download. Server replied HTTP code: " + httpConn.getResponseCode());
 }
 }
 
 private static HttpURLConnection request(String url) throws IOException
 {
 HttpURLConnection httpConn = (HttpURLConnection) new URL(url).openConnection();
 httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2");
 return httpConn;
 }
 
 private static void unzip(String zipFilePath, String destDir, boolean overwrite)
 {
 System.err.println("Unzipping to " + destDir);
 File dir = new File(destDir);
 // create output directory if it doesn't exist
 if (!dir.exists()) dir.mkdirs();
 FileInputStream fis;
 //buffer for read and write data to file
 byte[] buffer = new byte[4096];
 try
 {
 fis = new FileInputStream(zipFilePath);
 ZipInputStream zis = new ZipInputStream(fis);
 ZipEntry ze = zis.getNextEntry();
 while (ze != null)
 {
 String fileName = ze.getName();
 File newFile = new File(destDir + File.separator + fileName);
 if (overwrite || !newFile.exists())
 {
 if (ze.isDirectory())
 {
 //create directories for sub directories in zip
 newFile.mkdirs();
 }
 else
 {
 new File(newFile.getParent()).mkdirs();
 FileOutputStream fos = new FileOutputStream(newFile);
 int len;
 while ((len = zis.read(buffer)) > 0)
 {
 fos.write(buffer, 0, len);
 }
 fos.close();
 //close this ZipEntry
 zis.closeEntry();
 }
 }
 ze = zis.getNextEntry();
 }
 //close last ZipEntry
 zis.closeEntry();
 zis.close();
 fis.close();
 new File(zipFilePath).delete();
 }
 catch (IOException e)
 {
 e.printStackTrace();
 }
 }
 }

## (2)ModelTrain类 ##

package com.cn.test.TextClassification;
    
    import com.hankcs.hanlp.classification.classifiers.IClassifier;
    import com.hankcs.hanlp.classification.classifiers.NaiveBayesClassifier;
    import com.hankcs.hanlp.classification.models.NaiveBayesModel;
    import com.hankcs.hanlp.corpus.io.IOUtil;
    
    import java.io.File;
    
    
    public class ModelTrain {
        /**
         * 搜狗文本分类语料库5个类目，每个类目下1000篇文章，共计5000篇文章
         */
        public static final String CORPUS_FOLDER = TestUtility.ensureTestData("搜狗文本分类语料库迷你版", "");
    
        /**
         * 模型保存路径
         */
        public static final String MODEL_PATH = "data/test/classification-model.ser";
    
    
        public static NaiveBayesModel trainOrLoadModel()
        {
            NaiveBayesModel model = (NaiveBayesModel) IOUtil.readObjectFrom(MODEL_PATH);
            if (model != null) return model;
    
            File corpusFolder = new File(CORPUS_FOLDER);
            if (!corpusFolder.exists() || !corpusFolder.isDirectory())
            {
                System.err.println("没有文本分类语料!");
                System.exit(1);
            }
            try{
                IClassifier classifier = new NaiveBayesClassifier(); // 创建分类器，更高级的功能请参考IClassifier的接口定义
                classifier.train(CORPUS_FOLDER);                     // 训练后的模型支持持久化，下次就不必训练了
                model = (NaiveBayesModel) classifier.getModel();
                IOUtil.saveObjectTo(model, MODEL_PATH);
            }catch (Exception e){
                e.printStackTrace();
            }
            return model;
        }
    }

## (3)InitOneObject类 ##

package com.cn.test.TextClassification;
    
    import com.hankcs.hanlp.classification.classifiers.IClassifier;
    import com.hankcs.hanlp.classification.classifiers.NaiveBayesClassifier;
    
    public class InitOneObject {
        public static final InitOneObject instance= new InitOneObject();
        //获取classifier对象,训练后的模型支持持久化，下次就不必训练了.
        public IClassifier classifier = new NaiveBayesClassifier(ModelTrain.trainOrLoadModel());
    }

## (4)TrainMain类 ##

package com.cn.test.TextClassification;
    
    import com.hankcs.hanlp.classification.classifiers.IClassifier;
    
    import java.io.IOException;
    
    public class TrainMain {
        public static void main(String[] args) throws IOException
        {
            //IClassifier classifier = new NaiveBayesClassifier(trainOrLoadModel());
            IClassifier classifier= InitOneObject.instance.classifier;
            predict(classifier, "C罗获2018环球足球奖最佳球员 德尚荣膺最佳教练");
            predict(classifier, "英国造航母耗时8年仍未服役 被中国速度远远甩在身后");
            predict(classifier, "研究生考录模式亟待进一步专业化");
            predict(classifier, "如果真想用食物解压,建议可以食用燕麦");
            predict(classifier, "锄禾日当午，汗滴禾下土");
        }
    
        private static void predict(IClassifier classifier, String text)
        {
            System.out.printf("《%s》 属于分类 【%s】\n", text, classifier.classify(text));
        }
    }

# 4、运行结果 #

模式:训练集
    文本编码:UTF-8
    根目录:C:\MyselfApplication\MyProject\HanLp\data\test\搜狗文本分类语料库迷你版
    加载中...
    [体育]...100.00% 1000 篇文档
    [健康]...100.00% 1000 篇文档
    [军事]...100.00% 1000 篇文档
    [教育]...100.00% 1000 篇文档
    [汽车]...100.00% 1000 篇文档
    耗时 15868 ms 加载了 5 个类目,共 5000 篇文档
    原始数据集大小:5000
    使用卡方检测选择特征中...耗时 189 ms,选中特征数:18156 / 80986 = 22.42%
    贝叶斯统计结束
    《C罗获2018环球足球奖最佳球员 德尚荣膺最佳教练》 属于分类 【体育】
    《英国造航母耗时8年仍未服役 被中国速度远远甩在身后》 属于分类 【军事】
    《研究生考录模式亟待进一步专业化》 属于分类 【教育】
    《如果真想用食物解压,建议可以食用燕麦》 属于分类 【健康】
    《锄禾日当午，汗滴禾下土》 属于分类 【健康】

[20201016165017777.png]: /images/20221123/6f1dd91749854dce93eaa1349a9e83b5.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2h5al9raW5n_size_16_color_FFFFFF_t_70]: /images/20221123/d8bf0e3324234b8bacf536f35a5f8b5c.png