java 读取hdfs上的文件内容

雨点打透心脏的1/2处 2024-04-01 11:18 89阅读 0赞

import com.nature.component.process.vo.DebugDataResponse;
    import org.apache.commons.lang3.StringUtils;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FSDataInputStream;
    import org.apache.hadoop.fs.FileStatus;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.slf4j.Logger;
     
    import java.io.*;
    import java.net.URI;
    import java.util.*;
     
    /**
     * 操作hdfs文件的工具类
     */
     
    public class HdfsUtils {
        
     
        private static Logger logger = LoggerUtil.getLogger();
     
        /**
         * 读取hdfs指定路径的内容
         */
        public static String gethdfsData(String hdfsPath){
        
            String result = "";
            if(com.nature.third.utils.StringUtils.isNotEmpty(hdfsPath)){
        
                Path path = new Path(hdfsPath);
                Configuration configuration = new Configuration();
                FSDataInputStream fsDataInputStream = null;
                FileSystem fileSystem = null;
                BufferedReader br = null;
                // 定义一个字符串用来存储文件内容
                try {
        
                    fileSystem = path.getFileSystem(configuration);
                    fsDataInputStream = fileSystem.open(path);
                    br = new BufferedReader(new InputStreamReader(fsDataInputStream));
                    String str2;
                    while ((str2 = br.readLine()) != null) {
        
                        // 遍历抓取到的每一行并将其存储到result里面
                        result += str2 + "\n";
                    }
                } catch (IOException e) {
        
                    e.printStackTrace();
                } finally {
        
                    if(br!=null){
        
                        try {
        
                            br.close();
                        } catch (IOException e) {
        
                            e.printStackTrace();
                        }
                    }
                    if(fsDataInputStream!=null){
        
                        try {
        
                            fsDataInputStream.close();
                        } catch (IOException e) {
        
                            e.printStackTrace();
                        }
                    }
                    if(fileSystem!=null){
        
                        try {
        
                            fileSystem.close();
                        } catch (IOException e) {
        
                            e.printStackTrace();
                        }
                    }
                }
                logger.debug("文件内容:" + result);
            }
            return result;
        }
     
    }

核心方法：

Path path = new Path("hdfs://192.168.1.111:9000/test/1234log");
    Configuration configuration = new Configuration();
    FileSystem fileSystem = path.getFileSystem(configuration);
    FSDataInputStream fsDataInputStream = fileSystem.open(path);

注意我加的那些编码格式，可以防止中文乱码！！  
这种方式即可以读取hdfs上面的文件也可以读取本地文件