Java API 读写 hdfs分布式文件系统（创建目录、读文件、写文件）

电玩女神 2022-03-09 19:50 774阅读 0赞

Java API 读写 hdfs 操作  
一、概述

操作hdfs分布式文件系统的类为FileSystem对象

1、创建文件夹（递归创建）：fs. mkdirs()

2、写文件  ： fs.create（new Path()），返回FSDataOutputStream对象

3、读文件  ： fs.open(new Path())  , 返回FSDataInputStream对象

二、代码

package com.example.hdfs;

import org.apache.commons.io.IOUtils;  
import org.apache.hadoop.conf.Configuration;  
import org.apache.hadoop.fs.FSDataInputStream;  
import org.apache.hadoop.fs.FSDataOutputStream;  
import org.apache.hadoop.fs.FileSystem;  
import org.apache.hadoop.fs.Path;

import java.net.URI;  
import java.util.logging.Logger;

public class TestMain \{

private static final Logger logger = Logger.getLogger("com.example.hdfs.TestMain");

public static void main(String\[\] args) throws Exception \{  
        //HDFS URI\}  
        String hdfsuri = "hdfs://localhost:8020/";

String path="/user/hdfs/example/hdfs/";  
        String fileName="hello.csv";  
        String fileContent="hello;world";

// ====== Init HDFS File System Object  
        Configuration conf = new Configuration();  
        // Set FileSystem URI  
        conf.set("fs.defaultFS", hdfsuri);  
        // Because of Maven  
        conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());  
        conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());  
        // 设置hadoop应用程序执行用户  
        System.setProperty("HADOOP\_USER\_NAME", "hdfs");  
        System.setProperty("hadoop.home.dir", "/");  
        //创建 FileSystem - HDFS  
        FileSystem fs = FileSystem.get(URI.create(hdfsuri), conf);

//创建文件目录，如果不存在  
        Path workingDir=fs.getWorkingDirectory();  
        Path newFolderPath= new Path(path);  
        if(!fs.exists(newFolderPath)) \{  
            // Create new Directory  
            fs.mkdirs(newFolderPath);  
            logger.info("Path "+path+" created.");  
        \}

//==== Write file  
        logger.info("Begin Write file into hdfs");  
        //Create a path  
        Path hdfswritepath = new Path(newFolderPath + "/" + fileName);  
        //Init output stream  
        FSDataOutputStream outputStream=fs.create(hdfswritepath);  
        //Cassical output stream usage  
        outputStream.writeBytes(fileContent);  
        outputStream.close();  
        logger.info("End Write file into hdfs");

//==== Read file  
        logger.info("Read file into hdfs");  
        //Create a path  
        Path hdfsreadpath = new Path(newFolderPath + "/" + fileName);  
        //Init input stream  
        FSDataInputStream inputStream = fs.open(hdfsreadpath);  
        //Classical input stream usage  
        String out= IOUtils.toString(inputStream, "UTF-8");  
        logger.info(out);  
        inputStream.close();  
        fs.close();

\}  
\}