mapreduce打包提交执行wordcount案例

深藏阁楼爱情的钟 2023-09-26 21:57 124阅读 0赞

### 文章目录 ###

*   *  一、源代码
     *   *  1. WordCountMapper类
         *  2. WordCountReducer类
         *  3. WordCountDriver类
         *  4. pom.xml
     *  二、相关操作和配置
     *   *  1. 项目打包
         *  2. 带参测试
         *  3. 上传打包后的jar包和测试文档
         *  4. 增大虚拟内存
         *  5.启动集群
         *  6.在hdfs上创建输入文件夹和上传测试文档Hello.txt
         *  7. 利用jar包在hdfs实现文本计数
         *  8. 查看计算统计结果

## 一、源代码 ##

### 1. WordCountMapper类 ###

package org.example.wordcounttemplate;
    
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    import java.io.IOException;
    
    public class WordCountMapper extends Mapper<LongWritable, Text,Text, IntWritable> {
        
    
        //新建输出文本对象(输出的key类型)
        private Text text = new Text();
        //新建输出IntWritable对象(输出的value类型)
        private IntWritable intWritable = new IntWritable( 1);
    
    
        /**
         * 重写map方法
         * @param key 文本的索引
         * @param value 文本值
         * @param context 上下文对象
         * @throws IOException
         * @throws InterruptedException
         */
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        
    
            //获取拆分后的一行文本
    
            //mysql mysql value value value
            String line = value.toString();
    
            //根据分隔符进行单词拆分
            String[] words = line.split( " ");
    
            //循环创建键值对
            for (String word : words){
        
    
                //输出key值设置
                text.set (word) ;
    
                //进行map输出
                //igeek igeek -> <igeek ,1> <igeek,1>
                context.write(text,intWritable);
            }
    
        }
    }

### 2. WordCountReducer类 ###

package org.example.wordcounttemplate;
    
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    import java.io.IOException;
    
    public class WordCountReducer extends Reducer<Text, IntWritable,Text, IntWritable> {
        
    
        //输出value对象
        private IntWritable valueOut = new IntWritable();
    
    
        /**
         * 重写reduce方法
         * @param key 单词值
         * @param values 单词出现的次数集合
         * @param context   上下文对象
         * @throws IOException
         * @throws InterruptedException
         */
        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        
    
            //每个单词出现的次数
            int sum= 0;
    
            //<igeek,(1,1)>
            for (IntWritable value : values){
        
    
            //累计单词出现的数量
                sum += value.get();
            }
    
            //进行封装
            valueOut.set(sum);
    
            // reduce输出
            context.write(key, valueOut);
    
    
        }
    }

### 3. WordCountDriver类 ###

package org.example.wordcounttemplate;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    import java.io.IOException;
    
    /**
     * 充当mapreduce任务的客户端，用于提交任务
     */
    
    public class WordCountDriver {
        
        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        
    //        1.获取配置信息，获取job对象实例
            Configuration conf=new Configuration();
            Job job=Job.getInstance(conf);
    
    //        2.关联本Driver得jar路径
            job.setJarByClass(WordCountDriver.class);
    
    //        3.关联map和reduce
            job.setMapperClass(WordCountMapper.class);
            job.setReducerClass(WordCountReducer.class);
    
    //        4.设置map得输出kv类型
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(IntWritable.class);
    
    //        5.设置最终输出得kv类型
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(IntWritable.class);
    
    //        6.设置输入和输出路径
            FileInputFormat.setInputPaths(job,new Path(args[0]));
            FileOutputFormat.setOutputPath(job,new Path(args[1]));
    
    //        7.提交job
            boolean result=job.waitForCompletion(true);
            System.out.println(result?"任务提交成功":"任务提交失败");
    
        }
    
    
    }

### 4. pom.xml ###

重点是更改添加打包插件依赖

<plugins>
        <plugin>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.6.1</version>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
            </configuration>
        </plugin>
        <plugin>
            <artifactId>maven-assembly-plugin</artifactId>
            <configuration>
                <descriptorRefs>
                    <descriptorRef>jar-with-dependencies</descriptorRef>
                </descriptorRefs>
            </configuration>
            <executions>
                <execution>
                    <id>make-assembly</id>
                    <phase>package</phase>
                    <goals>
                        <goal>single</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>
    </plugins>

pom.xml文件内容如下：

<?xml version="1.0" encoding="UTF-8"?>
    
    <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
      <modelVersion>4.0.0</modelVersion>
    
      <groupId>org.example</groupId>
      <artifactId>mapreduce_demo</artifactId>
      <version>1.0-SNAPSHOT</version>
    
      <name>mapreduce_demo</name>
      
      <url>http://www.example.com</url>
    
      <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
      </properties>
    
      <dependencies>
        
        <dependency>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-client</artifactId>
          <version>3.1.3</version>
        </dependency>
    
      </dependencies>
      <build>
        <plugins>
          <plugin>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.6.1</version>
            <configuration>
              <source>1.8</source>
              <target>1.8</target>
            </configuration>
          </plugin>
          <plugin>
            <artifactId>maven-assembly-plugin</artifactId>
            <configuration>
              <descriptorRefs>
                <descriptorRef>jar-with-dependencies</descriptorRef>
              </descriptorRefs>
            </configuration>
            <executions>
              <execution>
                <id>make-assembly</id>
                <phase>package</phase>
                <goals>
                  <goal>single</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    
    </project>

## 二、相关操作和配置 ##

### 1. 项目打包 ###

![在这里插入图片描述][252c1e10aad6493b9292cfe6f9d9b29b.png]

### 2. 带参测试 ###

![在这里插入图片描述][f2315a685fc94e5888b731174cfd2998.png]  
![在这里插入图片描述][fcd514cc89e84250a87944e09bf39253.png]

![在这里插入图片描述][1b64937520b64a64927f8e1498ab1dad.png]

在本地执行成功：

![在这里插入图片描述][8a7dd37ece524244bc996189808a47e1.png]  
![在这里插入图片描述][8ec01dd078064eca92c741de2f070bc3.png]

### 3. 上传打包后的jar包和测试文档 ###

上传打包后的带依赖jar包（第二个）和测试文档Hello.txt 到linux系统及hdfs上

cd /opt/jar/
    ll

jar包改名：

mv mapreduce_demo-1.0-SNAPSHOT-jar-with-dependencies.jar wordcount.jar
    ll

![在这里插入图片描述][6ed67813dca54924ad3750fa7b9445a6.png]

![在这里插入图片描述][ecc695ccde0641c7a4d3992cbcd72249.png]

cd /opt/file/
     ll

![在这里插入图片描述][864d6ca4709a4c669aeef093413e1acf.png]

### 4. 增大虚拟内存 ###

进行MapReduce操作时，可能会报溢出虚拟内存的错误

> beyond the 'VIRTUAL’memory limit.  
> Current usage: 32.7 MB of 1 GB physical memory used;  
> 2.3 GB of 2.1 GB virtual memory used. Killing container.

![在这里插入图片描述][02c27b06aad64e16b50ac19c5badbcb9.png]

解决:

在mapred-site.xml中添加如下内容

<property>
            <name>yarn.nodemanager.vmem-check-enabled</name>
            <value>false</value>
            <description>Whether virtual memory limits will be enforced for containers</description>
        </property>
    	
    	
    	
        <property>
            <name>yarn.nodemanager.vmem-pmem-ratio</name>
            <value>5</value>
            <description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
        </property>

cd /opt/softs/hadoop3.1.3/etc/hadoop/
    vim mapred-site.xml

![在这里插入图片描述][41ba68dc23a044f5a61be7900d8c6abc.png]

分发到另外两台服务器虚拟机

scp mapred-site.xml root@bigdata04:/opt/softs/hadoop3.1.3/etc/hadoop/
    
    scp mapred-site.xml root@bigdata05:/opt/softs/hadoop3.1.3/etc/hadoop/

### 5.启动集群 ###

[root@bigdata03 hadoop]# start-dfs.sh

[root@bigdata05 ~]# start-yarn.sh

![在这里插入图片描述][946c5531801a4b37b378630d7e7cb9bd.png]  
![在这里插入图片描述][20f7cb2ba28b416686550ff172293393.png]

### 6.在hdfs上创建输入文件夹和上传测试文档Hello.txt ###

hadoop fs -ls /
    hadoop fs -mkdir /input
    
    hadoop fs -put Hello.txt  /input
    hadoop fs -ls  /input

![在这里插入图片描述][e9e2f248fffe479aaf2830b34d5b03bf.png]

### 7. 利用jar包在hdfs实现文本计数 ###

cd /opt/jar/
     ll
     
     hadoop jar wordcount.jar org.example.wordcounttemplate.WordCountDriver /input/Hello.txt /output

注意：输出目录需不存在，让他执行命令时自行创建

![在这里插入图片描述][02693ec538f6456dac7a6a613d232198.png]  
![在这里插入图片描述][005ac0d81fe7493592e6dd90abf7d627.png]  
![在这里插入图片描述][c8dff60fc61e44d29e19a1727d9c789e.png]

### 8. 查看计算统计结果 ###

hadoop fs -ls  /output
    hadoop fs -cat  /output/part-r-00000

![在这里插入图片描述][405acac1a0b945fd999bcc96aba5b9d5.png]

![在这里插入图片描述][007bd90c1cf24d08b03904b473082fad.png]

对照文章：  
大数据作业4（含在本地实现wordcount案例）  
[https://blog.csdn.net/m0\_48170265/article/details/130029532?spm=1001.2014.3001.5501][https_blog.csdn.net_m0_48170265_article_details_130029532_spm_1001.2014.3001.5501]

[252c1e10aad6493b9292cfe6f9d9b29b.png]: https://img-blog.csdnimg.cn/252c1e10aad6493b9292cfe6f9d9b29b.png
[f2315a685fc94e5888b731174cfd2998.png]: https://img-blog.csdnimg.cn/f2315a685fc94e5888b731174cfd2998.png
[fcd514cc89e84250a87944e09bf39253.png]: https://img-blog.csdnimg.cn/fcd514cc89e84250a87944e09bf39253.png
[1b64937520b64a64927f8e1498ab1dad.png]: https://img-blog.csdnimg.cn/1b64937520b64a64927f8e1498ab1dad.png
[8a7dd37ece524244bc996189808a47e1.png]: https://img-blog.csdnimg.cn/8a7dd37ece524244bc996189808a47e1.png
[8ec01dd078064eca92c741de2f070bc3.png]: https://img-blog.csdnimg.cn/8ec01dd078064eca92c741de2f070bc3.png
[6ed67813dca54924ad3750fa7b9445a6.png]: https://img-blog.csdnimg.cn/6ed67813dca54924ad3750fa7b9445a6.png
[ecc695ccde0641c7a4d3992cbcd72249.png]: https://img-blog.csdnimg.cn/ecc695ccde0641c7a4d3992cbcd72249.png
[864d6ca4709a4c669aeef093413e1acf.png]: https://img-blog.csdnimg.cn/864d6ca4709a4c669aeef093413e1acf.png
[02c27b06aad64e16b50ac19c5badbcb9.png]: https://img-blog.csdnimg.cn/02c27b06aad64e16b50ac19c5badbcb9.png
[41ba68dc23a044f5a61be7900d8c6abc.png]: https://img-blog.csdnimg.cn/41ba68dc23a044f5a61be7900d8c6abc.png
[946c5531801a4b37b378630d7e7cb9bd.png]: https://img-blog.csdnimg.cn/946c5531801a4b37b378630d7e7cb9bd.png
[20f7cb2ba28b416686550ff172293393.png]: https://img-blog.csdnimg.cn/20f7cb2ba28b416686550ff172293393.png
[e9e2f248fffe479aaf2830b34d5b03bf.png]: https://img-blog.csdnimg.cn/e9e2f248fffe479aaf2830b34d5b03bf.png
[02693ec538f6456dac7a6a613d232198.png]: https://img-blog.csdnimg.cn/02693ec538f6456dac7a6a613d232198.png
[005ac0d81fe7493592e6dd90abf7d627.png]: https://img-blog.csdnimg.cn/005ac0d81fe7493592e6dd90abf7d627.png
[c8dff60fc61e44d29e19a1727d9c789e.png]: https://img-blog.csdnimg.cn/c8dff60fc61e44d29e19a1727d9c789e.png
[405acac1a0b945fd999bcc96aba5b9d5.png]: https://img-blog.csdnimg.cn/405acac1a0b945fd999bcc96aba5b9d5.png
[007bd90c1cf24d08b03904b473082fad.png]: https://img-blog.csdnimg.cn/007bd90c1cf24d08b03904b473082fad.png
[https_blog.csdn.net_m0_48170265_article_details_130029532_spm_1001.2014.3001.5501]: https://blog.csdn.net/m0_48170265/article/details/130029532?spm=1001.2014.3001.5501