5、eclipse + HDFS的文件操作

青旅半醒 2023-06-08 10:46 38阅读 0赞

### 文章目录 ###

*  1、文件上传
 *  2、文件下载
 *  3、删除文件或文件夹
 *  4、文件重命名
 *  5、文件详情查看
 *  6、文件或文件夹判断

eclipse + hadoop的开发环境搭建，请参考上一章节。

# 1、文件上传 #

把windows本地的文件上传到HDFS中，示例：

@Test
    public void testCopyFromLocalFile() throws IOException, InterruptedException, URISyntaxException { 
    	Configuration conf = new Configuration();
    	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.85.137:9000"), conf, "root");
    	fs.copyFromLocalFile(new Path("D:/tmp/myfile.txt"), new Path("/user/lzj"));
    	fs.close();
    	System.out.println("successfully!");
    }

# 2、文件下载 #

从HDFS下载文件到windows本地

@Test
    public void testCopyToLocalFile() throws IOException, InterruptedException, URISyntaxException { 
    	Configuration conf = new Configuration();
    	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.85.137:9000"), conf, "root");
    	fs.copyToLocalFile(new Path("/user/lzj/test.txt"), new Path("D:/tmp/"));
    	fs.close();
    	System.out.println("successfully!");
    }

# 3、删除文件或文件夹 #

删除HDFS系统中的文件或文件夹

@Test
    public void testDelete() throws IOException, InterruptedException, URISyntaxException { 
    	Configuration conf = new Configuration();
    	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.85.137:9000"), conf, "root");
    	/*如果删除的是文件，参数false或true不受影响；如果删除的是目录，true表示递归删除目录下文件*/
    	fs.delete(new Path("/user/lzj/myfile.txt"), false);
    	fs.close();
    	System.out.println("successfully!");
    }

# 4、文件重命名 #

重命名HDFS系统中的文件名

@Test
    public void testRename() throws IOException, InterruptedException, URISyntaxException { 
    	Configuration conf = new Configuration();
    	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.85.137:9000"), conf, "root");
    	fs.rename(new Path("/user/lzj/myfile.txt"), new Path("/user/lzj/myfile1.txt"));
    	fs.close();
    	System.out.println("successfully!");
    }

# 5、文件详情查看 #

查看HDFS文件系统中文件详情

@Test
    public void testFileDetails() throws IOException, InterruptedException, URISyntaxException { 
    	Configuration conf = new Configuration();
    	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.85.137:9000"), conf, "root");
    	RemoteIterator<LocatedFileStatus> iterators = fs.listFiles(new Path("/user/lzj"), true); /*true表示递归*/
    	int i = 0;
    	while(iterators.hasNext()) { 
    		System.out.println("-----第" + i++ + "个文件-----");
    		LocatedFileStatus fileStatus = iterators.next();
    		/*文件名称*/
    		System.out.println(fileStatus.getPath().getName());
    		/*文件长度*/
    		System.out.println(fileStatus.getLen());
    		/*文件权限*/
    		System.out.println(fileStatus.getPermission());
    		/*文件分组*/
    		System.out.println(fileStatus.getGroup());
    		/*文件存储块信息*/
    		BlockLocation[] blockLocations = fileStatus.getBlockLocations();
    		for(BlockLocation blockLocation : blockLocations) { 
    			/*获取存储块的主机节点*/
    			String[] hosts = blockLocation.getHosts();
    			for(String host : hosts) { 
    				System.out.println(host);
    			}
    		}
    	}
    	fs.close();
    	System.out.println("successfully");
    }

运行demo，输出如下：

-----第0个文件-----
    myfile.txt
    0
    rw-r--r--
    supergroup
    c172b0d0013a
    -----第1个文件-----
    myfile1.txt
    0
    rw-r--r--
    supergroup
    -----第2个文件-----
    test.txt
    12
    rw-r--r--
    supergroup
    c172b0d0013a
    1dadd673bf95
    successfully

# 6、文件或文件夹判断 #

判断HDFS系统上指定的目录下是文件还是文件夹

@Test
    public void testIsFileOrDirectory() throws IOException, InterruptedException, URISyntaxException { 
    	Configuration conf = new Configuration();
    	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.85.137:9000"), conf, "root");
    	FileStatus[] fileStatus = fs.listStatus(new Path("/user/lzj"));
    	for(FileStatus fileStatu : fileStatus) { 
    		if(fileStatu.isFile()) { 
    			System.out.println("f: " + fileStatu.getPath().getName());
    		}else { 
    			System.out.println("d: " + fileStatu.getPath().getName());
    		}
    	}
    	fs.close();
    	System.out.println("successfully");
    }

运行demo，输出如下：

f: myfile.txt
    f: myfile1.txt
    f: test.txt
    successfully