Java 对HDFS接口调用

矫情吗；* 2022-05-28 04:54 176阅读 0赞

使用HDFS提供的API来对分布式文件系统中的文件进行基本的操作，在linux下是不需要连接的，在windows下连接hdfs有三种方式：

Configuration  conf = new Configuration();
    //第一种方式
    conf.set("fs.defaultFS", "hdfs://10.49.85.152:9000");
    //第二种设置configuration方式
    conf.addResource(new Path("D:\\hadoop\\1.xml"));    
    FileSystem hdfs = FileSystem.get(conf);
    //第三种方式
    FileSystem hdfs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf);

--------------------

下面是对基本的API的调用，以供参考：

import java.io.FileNotFoundException;
    import java.io.IOException;
    import java.net.URI;
    import java.net.URISyntaxException;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FSDataInputStream;
    import org.apache.hadoop.fs.FSDataOutputStream;
    import org.apache.hadoop.fs.FileStatus;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    
    public class HDFSDemo2 { 
        static Configuration  conf;
        static FileSystem fs;
    
        static{
            conf = new Configuration();
            try {
                fs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf);
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (URISyntaxException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    
    // public HDFSDemo2(){ 
    // conf = new Configuration();
    // try { 
    // fs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf);
    // } catch (IOException e) { 
    // // TODO Auto-generated catch block
    // e.printStackTrace();
    // } catch (URISyntaxException e) { 
    // // TODO Auto-generated catch block
    // e.printStackTrace();
    // }
    // }
        //创建文件夹
        public static void mkdir(String name){
            boolean flag = false; 
            try {
                 if (!fs.exists(new Path(name+"1"))) {  
                     flag = fs.mkdirs(new Path(name));
                    }  
                System.out.println(flag);
                if(flag){
                    System.out.println("文件夹创建成功");
                }else{
                    System.out.println("文件夹创建失败");
                }
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }finally{
                close();
            }
        }
        //创建文件
        public static void createFile(String file, String text){
            FSDataOutputStream outputStream = null;
            byte[] arg0 = text.getBytes();
            try {
                outputStream = fs.create(new Path(file));
                outputStream.write(arg0, 0, arg0.length);
                outputStream.close();
                System.out.println("文件创建成功");
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }finally{
                close();
            }
        }
        //对文件重命名
        public static void rename(String oldName, String newName){
            boolean flag;
            try {
                flag = fs.rename(new Path(oldName), new Path(newName));
                if(flag){
                    System.out.println("文件重命名成功");
                }
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }finally{
                close();
            }
        }
    
    
        //判断文件是否存在
        public static void  existFile(String file){
            boolean flag;
            try {
                flag = fs.exists(new Path(file));
                if(flag){
                    System.out.println("存在");
                }else
                {
                    System.out.println("不存在");
                }
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }finally{
                close();
            }
        }
    
        //删除文件
        public static void deleteFile(String file){
            boolean flag;
            try {
                flag = fs.deleteOnExit(new Path(file));
                if(flag){
                    System.out.println("成功删除");
                }
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }finally{
                close();
            }
        }
    
    
        public static void close(){
            try {
                fs.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    
        //上传文件
        public static void uploadFile(String src, String dst){
            try {
                fs.copyFromLocalFile(false, new Path(src), new Path(dst));
                System.out.println("文件上传成功");
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }finally{
                close();
            }
        }
    
        //下载文件
        public static void downFile(String src, String dst){
            try {
                fs.copyToLocalFile(new Path(src), new Path(dst));
                System.out.println("文件下载成功");
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }finally{
                close();
            }
        }
    
        //读取文件
        public static String readFile(String file){
            String str = "";
            FSDataInputStream  inputStream = null;
            byte[] arg0 = new byte[1024];
            try {
                inputStream = fs.open(new Path(file));
                inputStream.read(arg0);
                str = new String(arg0);
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }finally{
                close();
            }
    
            return str;
        }
    
        //返回指定文件夹的文件列表
        public static void listName(String dir){
            try {
                FileStatus files[] = fs.listStatus(new Path(dir));
                for(FileStatus file : files){
                    System.out.println(file.getPath().getName() + "\t" + file.getLen());
                }
            } catch (FileNotFoundException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IllegalArgumentException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }finally{
                close();
            }
    
    
        }
    
        public static void main(String[] args) throws IOException {
            // TODO Auto-generated method stub
    // HDFSDemo2 hdfs = new HDFSDemo2();
            HDFSDemo2.mkdir("/test/t2");
    // HDFSDemo2.createFile("/test/3.txt", "fdsf23432432违法所得");
    // HDFSDemo2.createFile(args[0], args[1]);
    // HDFSDemo2.rename("/test/test2", "/test/test1");
    // HDFSDemo2.existFile("/test/3.txt");
    // HDFSDemo2.deleteFile("/test/5.txt");
    // HDFSDemo2.downFile(args[0], args[1]);
    // String str = HDFSDemo2.readFile("/test/3.txt");
    // System.out.println(str);
    // HDFSDemo2.listName("/test");
    
        }
    
    }