5.7 HBase和MapReduce整合

小咪咪 2022-05-23 12:47 152阅读 0赞

读hdfs数据写到HBase中：

![20180609092940295][]

注意hadoop和HBase的包都需要导入，

先创建表

WordCountMapper：

<table style="table-layout:fixed;white-space:nowrap;width:0px;">    
 <tbody> 
  <tr style="height:116px;"> 
   <td style="color:rgb(57,57,57);border-color:rgb(167,167,167);white-space:pre-wrap;"> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">package com.laoxiao.mr.hbase;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import java.io.IOException;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.io.IntWritable;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.io.LongWritable;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.io.Text;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.mapreduce.Mapper;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">/**</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;* </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;* @author root</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;* 定义map任务输入和输出数据类型。</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;*</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;*/</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">public class WordCountMapper extends Mapper&lt;LongWritable, Text, Text, IntWritable&gt;{ </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;/**</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; * map方法是一行数据调用一次。每一次调用传入一行数据。该行数据的下标位为key。内容为value</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; */</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;protected void map(LongWritable key, Text value,</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;Context context)</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;throws IOException, InterruptedException { </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;String[] words =value.toString().split(" ");</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;for (int i = 0; i &lt; words.length; i++) { </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;String w = words[i];</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;Text outkey =new Text(w);</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;IntWritable outvalue=new IntWritable(1);</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;context.write(outkey, outvalue);</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; }</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;}</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">}</span> 
    </div></td> 
  </tr> 
 </tbody> 
</table>

WordCountReducer：

<table style="table-layout:fixed;white-space:nowrap;width:0px;">    
 <tbody> 
  <tr style="height:112px;"> 
   <td style="color:rgb(57,57,57);border-color:rgb(167,167,167);white-space:pre-wrap;"> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">package com.laoxiao.mr.hbase;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import java.io.IOException;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.hbase.client.Put;</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.hbase.io.ImmutableBytesWritable;</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.hbase.mapreduce.TableReducer;</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.io.IntWritable;</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.io.Text;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">public class WordCountReducer extends TableReducer&lt;Text, IntWritable, ImmutableBytesWritable&gt;{ </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;/*</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; * 该方法每一组调用一次。</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; */</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;protected void reduce(Text key, Iterable&lt;IntWritable&gt; arg1,</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;Context context)</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;throws IOException, InterruptedException { </span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; int sum =0;</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; for(IntWritable i :arg1){ </span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;sum=sum+i.get();</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; }</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; Put put = new Put(key.toString().getBytes());</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; put.add("cf".getBytes(), "count".getBytes(), (sum+"").getBytes());</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; context.write(null, put);</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;}</span> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">}</span> 
    </div></td> 
  </tr> 
 </tbody> 
</table>

RunJob：

<table style="table-layout:fixed;white-space:nowrap;width:0px;">    
 <tbody> 
  <tr style="height:115px;"> 
   <td style="color:rgb(57,57,57);border-color:rgb(167,167,167);white-space:pre-wrap;"> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">package com.laoxiao.mr.hbase;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.conf.Configuration;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.fs.Path;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.io.IntWritable;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.io.Text;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.mapreduce.Job;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">public class RunJob { </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;public static void main(String[] args) throws Exception { </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; Configuration config =new Configuration();</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; config.set("fs.defaultFS", "hdfs://node1:9000");</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; config.set("yarn.resourcemanager.hostname", "node1");</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">// config.set("mapred.jar", "C:\\Users\\Administrator\\Desktop\\wc.jar");</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; // hbase zk集群</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; config.set("hbase.zookeeper.quorum", "node1");</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">// try { </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;Job job =Job.getInstance(config);</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;job.setJobName("word count");</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;job.setJarByClass(RunJob.class);//job的入口类</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;job.setMapperClass(WordCountMapper.class);</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;job.setMapOutputKeyClass(Text.class);</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;job.setMapOutputValueClass(IntWritable.class);</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;String tableName = "wc";</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;// 本地方式运行 参数addDependencyJars 需要设置为false 如果集群模式运行 保持true即可</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">// TableMapReduceUtil.initTableReducerJob(tableName, WordCountReducer.class, job);//集群运行</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;TableMapReduceUtil.initTableReducerJob(tableName, WordCountReducer.class, job,</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp; &nbsp;null, null, null, null, false);</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp; &nbsp;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;Path inputPath = new Path("/usr/input/wc.txt");</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;FileInputFormat.addInputPath(job, inputPath);</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;boolean f= job.waitForCompletion(true);</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;if(f){ </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp; System.out.println("job执行成功！");</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; &nbsp;}</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">// } catch (Exception e) { </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">// // TODO Auto-generated catch block</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">// e.printStackTrace();</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">// }</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp; </span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">&nbsp;}</span> 
    </div> 
    <div> 
     <br> 
    </div> 
    <div> 
     <span style="font-family:'Microsoft YaHei', STXihei;color:rgb(0,0,0);background-color:transparent;">}</span> 
    </div></td> 
  </tr> 
 </tbody> 
</table>

[20180609092940295]: /images/20220523/82a3701de1bd47adb1f2e4bfa5e9a608.png