多线程处理文件

川长思鸟来 2022-04-12 13:09 282阅读 0赞

package com.geotmt.zxw.utils;
    
    import com.google.common.collect.Lists;
    import lombok.extern.slf4j.Slf4j;
    
    import java.io.*;
    import com.google.common.io.Files;
    
    import java.util.Collections;
    import java.util.List;
    import java.util.concurrent.FutureTask;
    import java.util.stream.Collectors;
    
    /**
     *
     * Created by geo on 2018/12/3. */
    @Slf4j
    public class FileUtil2 {
    
        /**
         * 读取文件（前提内存允许）
         *
         * @param filePathAndName 文件路径及名称
         */
        static List<String> readFileAndSpilt(String filePathAndName) {
            String encoding = "utf-8";
            List<String> txtList = Lists.newArrayList();
            File file = new File(filePathAndName);
            try {
                if (file.isFile() && file.exists()) { //判断文件是否存在
                    InputStreamReader read = new InputStreamReader(new FileInputStream(file), encoding);//考虑到编码格式
                    BufferedReader bufferedReader = new BufferedReader(read);
                    String lineTxt;
                    while ((lineTxt = bufferedReader.readLine()) != null) {
                        txtList.add(lineTxt) ;
                    }
                    read.close();
                } else {
                    log.info("找不到指定的文件");
                }
            } catch (Exception e) {
                log.info("读取文件内容出错:",e);
            }
            return txtList ;
        }
    
        /**
         * 写文件
         * @param txtList 文件
         * @param outFilePathAndName 输出路径
         */
        static void writeTxt(List<String> txtList,String outFilePathAndName){
            File spiltFilePathAndNameFile = new File(outFilePathAndName) ;
            File path = new File(outFilePathAndName.substring(0,outFilePathAndName.lastIndexOf("/"))) ;
            if(!path.exists()){
                try {
                    boolean newFile = path.mkdirs();
                    log.info("创建目录[{}]结果[{}].",path.getPath(),newFile);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            try {
                boolean newFile = spiltFilePathAndNameFile.createNewFile() ;
                log.info("创建文件[{}]结果[{}].",outFilePathAndName,newFile);
            } catch (IOException e) {
                e.printStackTrace();
            }
            // 数组转换文件，按照行分割
            StringBuffer sb = new StringBuffer() ;
            txtList.forEach(v -> sb.append(v).append("\n"));
            smallFileWrite(outFilePathAndName,sb.toString()) ;
        }
    
        /**
         * 演示向文件中写入字节流
         *
         * @param fileName 要写入文件的文件名
         * @param contents 要写入的文件内容
         */
        static void smallFileWrite(final String fileName, final String contents){
            try{
                final File newFile = new File(fileName);
                Files.write(contents.getBytes(), newFile);
            }catch (Exception e){
                log.info(  "ERROR trying to write to file [{}]",fileName,e);
            }
        }
    
        /**
         * 其他操作，即睡10ms
         * @param line 文本行
         * @return 处理过的文本行
         */
        static String otherOper(String line){
            try {
                Thread.sleep(10);
            }catch (Exception e){
                e.printStackTrace();
            }
            return line + System.currentTimeMillis() ;
        }
    
        /**
         * 处理方法
         *
         * @param stringList List<String>
         * @return List<String>
         */
        static List<String> handle(List<String> stringList,List<String> resultList) {
            FutureTask<List<String>> task = new FutureTask<>(()->stringList.stream().map(FileUtil2::otherOper).collect(Collectors.toList()));
            new Thread(task).start() ;
            try {
                resultList.addAll(task.get()) ;
                return resultList;
            } catch (Exception e) {
                e.printStackTrace();
                return null ;
            }
        }
    
        public static void main(String[] args) {
            int thredNum = 10 ; // 线程数
            String filePathAndName = "" ; // 文件路径
            String outFilePathAndName = ""; // 输出路径
            List<String> resultList = Collections.synchronizedList(Lists.newArrayList()) ;
            List<String> txtList = readFileAndSpilt(filePathAndName) ; // 读取文本，前提是文件不能超出堆的承受能力
            List<List<String>> txtLists = Lists.partition(txtList, thredNum); // 线程数切割
            txtLists.forEach(list ->handle(list,resultList)); // 处理文本
            writeTxt(txtList,outFilePathAndName) ; // 输出结果
        }
    
    
    }