springboot整合hadoop

落日映苍穹つ 2023-09-28 21:48 14阅读 0赞

### 首先引入maven依赖 ###

<dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-client</artifactId>
                <version>3.1.4</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-hdfs</artifactId>
                <version>3.1.4</version>
            </dependency>

### 代码 ###

package com.xiaowuqin.hdfs;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.springframework.stereotype.Component;
    import java.io.IOException;
    import java.net.URI;
    import java.net.URISyntaxException;
    
    @Component
    public class HdfsClient {
        //创建目录
        private FileSystem fs;
        public void init() throws URISyntaxException, IOException, InterruptedException {
            URI uri = new URI("hdfs://hadoop102:8020");
            Configuration configuration = new Configuration();
            String user="root";
            fs = FileSystem.get(uri, configuration,user);//获取一个客户端对象
        }
        public void makeDir(String dir) throws URISyntaxException, IOException, InterruptedException {
            init();
            Path path = new Path(dir);
            fs.mkdirs(path);
            fs.close();
        }
        //上传文件
        public void putFile(String fileName,String to) throws InterruptedException, IOException, URISyntaxException {
            init();
            //第一个参数：上传后是否删除本地源文件
            //第二个参数：是否允许覆盖
            //第三个参数：源文件路径
            //第四个参数: 目标文件路径
            fs.copyFromLocalFile(false,true,
                    new Path(fileName),
                    new Path(to));
            fs.close();
        }
        public void uploadAll() throws InterruptedException, IOException, URISyntaxException {
            for(int i=1;i<=30;i++){
                if(i<=9) {
                    String path="D:\迅雷下载\2018年数据可视分析挑战赛-挑战1-数据\chinavis2018挑战一数据\2017-11-0"+i;
                    String to="/BigWorks/2017-11-0" + i;
                    makeDir(to);
                    putFile(path+"\checking.csv",to);
                    putFile(path+"\email.csv",to);
                    putFile(path+"\login.csv",to);
                    putFile(path+"\tcpLog.csv",to);
                    putFile(path+"\weblog.csv",to);
    
                }else {
                    String path="D:\迅雷下载\2018年数据可视分析挑战赛-挑战1-数据\chinavis2018挑战一数据\2017-11-"+i;
                    String to="/BigWorks/2017-11-" + i;
                    makeDir(to);
                    putFile(path+"\checking.csv",to);
                    putFile(path+"\email.csv",to);
                    putFile(path+"\login.csv",to);
                    putFile(path+"\tcpLog.csv",to);
                    putFile(path+"\weblog.csv",to);
    
                }
            }
        }
    
    }