Windows下使用Java API操作HDFS的常用方法

电玩女神 2022-09-03 00:08 161阅读 0赞

# 场景 #

Windows下配置Hadoop的Java开发环境以及用Java API操作HDFS：

[https://blog.csdn.net/BADAO\_LIUMANG\_QIZHI/article/details/119379055][https_blog.csdn.net_BADAO_LIUMANG_QIZHI_article_details_119379055]

在上面将Hadoop的开发环境搭建起来之后，使用Java API 简单输出了文件目录。

那么对应HDFS的常用文件的操作还有哪些。

注：

博客：  
[https://blog.csdn.net/badao\_liumang\_qizhi][https_blog.csdn.net_badao_liumang_qizhi]  
关注公众号  
霸道的程序猿  
获取编程相关电子书、教程推送与免费下载。

# 实现 #

1、获取HDFS文件系统

/**
         * 获取HDFS文件系统
         * @return
         * @throws IOException
         */
        public static FileSystem getFileSystem() throws IOException {
            Configuration configuration = new Configuration();
            configuration.set("fs.defaultFS","hdfs://192.168.148.128:9000");
            configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
            System.setProperty("HADOOP_USER_NAME","root");
            FileSystem fileSystem = FileSystem.get(configuration);
            return fileSystem;
        }

2、获取HDFS中所有的dataNodes

/**
         * 获取HDFS中所有的dataNode
         * @return
         * @throws IOException
         */
        public static void listDataNodeInfo() throws IOException {
            FileSystem fs = getFileSystem();
            DistributedFileSystem hdfs = (DistributedFileSystem) fs;
            DatanodeInfo[] dataNodeStats =hdfs.getDataNodeStats();
            String[] names = new String[dataNodeStats.length];
            System.out.println("HDFS中所有的dataNode:");
            for(int i=0;i<names.length;i++)
            {
                names[i]=dataNodeStats[i].getHostName();
                System.out.println(names[i]);
            }
        }

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70][]

3、创建文件夹

/**
         * 创建文件夹
         * @return
         * @throws IOException
         */
        public static void mkdir() throws IOException {
            FileSystem fs = getFileSystem();
            Path srcPath = new Path("/newDir");
            boolean isok = fs.mkdirs(srcPath);
            if(isok)
            {
                System.out.printf("创建文件夹成功");
            }else
            {
                System.out.printf("创建文件夹失败");
            }
            fs.close();
        }

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 1][]

4、删除文件夹

/**
         * 删除文件夹
         * @return
         * @throws IOException
         */
        public static void deletedir() throws IOException {
            FileSystem fs = getFileSystem();
            Path srcPath = new Path("/newDir");
            boolean isok = fs.deleteOnExit(srcPath);
            if(isok)
            {
                System.out.printf("删除文件夹成功");
            }else
            {
                System.out.printf("删除文件夹失败");
            }
            fs.close();
        }

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 2][]

5、判断文件是否存在

/**
         * 判断文件是否存在
         * @return
         * @throws IOException
         */
        public static void checkFileExists() throws IOException {
            FileSystem fs = getFileSystem();
            Path srcPath = new Path("/badao.txt");
            boolean isexist = fs.exists(srcPath);
            if(isexist)
            {
                System.out.printf("文件存在");
            }else
            {
                System.out.printf("文件不存在");
            }
        }

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 3][]

6、上传文件

/**
         * 上传文件
         * @return
         * @throws IOException
         */
        public static void uploadFile() throws IOException {
    
            FileSystem fs = getFileSystem();
            //源路径
            Path srcPath = new Path("D:\\windows.txt");
            //目标路径
            Path targetPath = new Path("/");
            fs.copyFromLocalFile(false,srcPath,targetPath);
            fs.close();
        }

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 4][]

7、下载文件

/**
         * 下载文件
         * @return
         * @throws IOException
         */
        public static void downloadFile() throws IOException {
    
            FileSystem fs = getFileSystem();
            //源路径
            Path targetPath = new Path("D:\\");
            //目标路径
            Path srcPath = new Path("/user/1.txt");
            fs.copyToLocalFile(srcPath,targetPath);
            fs.close();
        }

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 5][]

5、重命名文件

/**
         * 重命名文件
         * @return
         * @throws IOException
         */
        public static void renameFile() throws IOException {
    
            FileSystem fs = getFileSystem();
            //源路径
            Path oldPath = new Path("/windows.txt");
            //目标路径
            Path newPath = new Path("/centos.txt");
            boolean isok = fs.rename(oldPath,newPath);
            if(isok)
            {
                System.out.printf("重命名成功");
            }else
            {
                System.out.printf("重命名失败");
            }
            fs.close();
        }

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 6][]

6、遍历目录和文件

/**
         * 遍历目录和文件
         * @return
         * @throws IOException
         */
        public static void showDir(Path path) throws IOException {
    
            FileSystem fs = getFileSystem();
            DistributedFileSystem hdfs = (DistributedFileSystem) fs;
            FileStatus[] fileStatuses = hdfs.listStatus(path);
            if(fileStatuses.length>0)
            {
                for (FileStatus status:fileStatuses) {
                    Path f = status.getPath();
                    System.out.println(f.toString());
                    if(status.isDirectory())
                    {
                        FileStatus[] files = hdfs.listStatus(f);
                        if(files.length>0)
                        {
                            for (FileStatus file:files) {
                                showDir(file.getPath());
                            }
                        }
                    }
                }
            }
        }

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 7][]

[https_blog.csdn.net_BADAO_LIUMANG_QIZHI_article_details_119379055]: https://blog.csdn.net/BADAO_LIUMANG_QIZHI/article/details/119379055
[https_blog.csdn.net_badao_liumang_qizhi]: https://blog.csdn.net/badao_liumang_qizhi
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70]: /images/20220829/58a8bf02602b4c76ae05e1b1e565237e.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 1]: /images/20220829/348c053b5d5f4bbc95de28860ad3f6ec.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 2]: /images/20220829/0c3d735e06034986b430366ddcc91175.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 3]: /images/20220829/72b5ea93111d420b9c7b79838cc38770.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 4]: /images/20220829/90fe81b981424c77b1dc70f1b33d3d2e.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 5]: /images/20220829/c392b864adbc4f94932be1f1fc68449b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 6]: /images/20220829/0c6599142e5747cbbb26122220e36193.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0JBREFPX0xJVU1BTkdfUUlaSEk_size_16_color_FFFFFF_t_70 7]: /images/20220829/919a4d89733e47e0935fbdc633ba2183.png