HDFS学习之FileSystem

古城微笑少年丶 2022-08-22 12:26 208阅读 0赞

> 本地访问HDFS最主要的方式是使用HDFS提供的Java API，其他的访问方式也均是建立在这些API之上的。
> 
> Hadoop2.6.4API文档：[http://hadoop.apache.org/docs/r2.6.4/api/index.html][http_hadoop.apache.org_docs_r2.6.4_api_index.html]
> 
> HdfsDao中的方法：

> > (1). 文件上传、下载；
> 
> > (2). 创建文件、文件夹；
> 
> > (3). 删除文件、文件夹；
> 
> > (4). 重命名文件或文件夹；
> 
> > (5). 列出目录下的文件；
> 
>     import java.io.BufferedInputStream;
>     import java.io.File;
>     import java.io.FileInputStream;
>     import java.io.IOException;
>     import java.io.InputStream;
>     import java.io.OutputStream;
>     
>     import org.apache.hadoop.conf.Configuration;
>     import org.apache.hadoop.fs.FSDataOutputStream;
>     import org.apache.hadoop.fs.FileStatus;
>     import org.apache.hadoop.fs.FileSystem;
>     import org.apache.hadoop.fs.Path;
>     import org.apache.hadoop.io.IOUtils;
>     /**
>      *  HDFS JavaAPI uses
>      * @author baalhuo
>      *
>      */
>     public class HdfsDao {
>     	private Configuration conf =null;
>     	public HdfsDao(){
>     		conf =new Configuration();
>     		conf.addResource(new Path("/hadoop/etc/hadoop/core-site.xml"));
>     	}
>     	public HdfsDao(Configuration conf){
>     		this.conf =conf;
>     	}	
>     	public static void main(String[] args) throws IOException {
>     		HdfsDao hdfsDao = new HdfsDao();
>     		hdfsDao.lsFile("/user/root/");
>     	}	
>     	/*Upload file to HDFS*/
>     	public boolean  uploadFile(String path,String localfile){
>     		File file=new File(localfile);
>     		if (!file.isFile()) {
>     			System.out.println(file.getName());
>     			return false;
>     		}
>     		try {			
>     			FileSystem fs =FileSystem.get(conf);
>     			InputStream in = new BufferedInputStream(new FileInputStream(localfile));
>     			OutputStream out = fs.create(new Path(path+"/"+file.getName()));
>     			IOUtils.copyBytes(in, out, 4096, true);
>     			return true;
>     		} catch (IOException e) {
>     			e.printStackTrace();
>     		}
>     		return false;
>     	}
>     	/* Download file from HDFS */
>     	public boolean downloadFile(String hadfile,String localPath){
>     		try {
>     			FileSystem fs =FileSystem.get(conf);
>     			fs.copyToLocalFile(new Path(hadfile), new Path(localPath));
>     			return true;
>     		} catch (IOException e) {
>     			e.printStackTrace();
>     		}
>     		return false;
>     	}
>     	/*Create a folder*/
>     	public void mkdirFolder(String folder) throws IOException {
>             Path path = new Path(folder);
>             FileSystem fs = FileSystem.get(conf);
>             if (!fs.exists(path)) {
>                 fs.mkdirs(path);
>                 System.out.println("Create: " + folder);
>             }
>             fs.close();
>         }
>         /* create a file by content */
>         public void createFile(String file, String content) throws IOException {
>             FileSystem fs = FileSystem.get(conf);
>             byte[] buff = content.getBytes();
>             FSDataOutputStream os = null;
>             try {
>                 os = fs.create(new Path(file));
>                 os.write(buff, 0, buff.length);
>                 System.out.println("Create: " + file);
>             } finally {
>                 if (os != null)
>                     os.close();
>             }
>             fs.close();
>         }
>     	/* Delete file from HDFS */
>     	public boolean deleteFile(String hadfile){
>     		try {
>     			FileSystem hadoopFS =FileSystem.get(conf);
>     			Path hadPath=new Path(hadfile);
>     			Path p=hadPath.getParent();
>     			boolean rtnval= hadoopFS.delete(hadPath, true);
>     			FileStatus[] hadfiles= hadoopFS.listStatus(p);
>     			for(FileStatus fs :hadfiles){
>     				System.out.println(fs.toString());
>     			}
>     			return rtnval;
>     		} catch (IOException e) {
>     			e.printStackTrace();
>     		}
>     		return false;
>     	}
>     	/* Delete a folder */
>         public void deleteFolder(String folder) throws IOException {
>             Path path = new Path(folder);
>             FileSystem fs = FileSystem.get(conf);
>             fs.deleteOnExit(path);
>             System.out.println("Delete: " + folder);
>             fs.close();
>         }
>     	/* Rename a file of HDFS */
>     	public void renameFile(String src, String dst) throws IOException {
>             Path name1 = new Path(src);
>             Path name2 = new Path(dst);
>             FileSystem fs = FileSystem.get(conf);
>             fs.rename(name1, name2);
>             System.out.println("Rename from " + src + " to " + dst);
>             fs.close();
>         }
>     	/* Show file list */
>     	public void lsFile(String folder) throws IOException {
>             Path path = new Path(folder);
>             FileSystem fs = FileSystem.get(conf);
>             FileStatus[] list = fs.listStatus(path);
>             System.out.println("---ls: " + folder);
>             if(list.length>0){
>             	  for (FileStatus f : list) {
>                       System.out.printf("Path: %s, idFile: %s, Size: %d\n", f.getPath(), f.isFile(), f.getLen());
>                       if(!f.isFile()){
>                       	String temppath = f.getPath().toString();
>                       	String[] tempfolder = temppath.split("9000");
>                       	lsFile(tempfolder[1]);
>                       }
>                   }
>             }
>             fs.close();
>         }
>     }
> 
>   
>   
>

[http_hadoop.apache.org_docs_r2.6.4_api_index.html]: http://hadoop.apache.org/docs/r2.6.4/api/index.html