HDPCD-Java-复习笔记（3）-lab

谁借莪１个温暖的怀抱￠ 2022-06-07 03:58 197阅读 0赞

**Java Lab Booklet**

**Lab: Understanding Block Storage**

1.The block size needs to be at least （1,048,576 bytes） according to the dfs.namenode.fs - limits.min -block-size property。

2.The block size must be a multiple of 512 bytes (the checksum size).

**View the number of blocks**

**hdfs fsck /user/root/stocks.csv**

\-files \--- the names of the files on the DataNodes.

\-blocks \--- the block IDs of the file.

\-locations \--- the IP addresses of the DataNodes.

**Lab: Configuring a Hadoop Development Environment**

hdfs dfsadmin -report \--- Verify DataNodes in cluster.

yarn node -list \-- Verify NodeManagers in cluster.

**Lab: Putting Files in HDFS with Java**

Configuration configuration = new Configuration();

FileSystem fs = FileSystem.get(configuration);  
Path path = new Path("counties");  
Path localPath = null;  
Path destPath = null;  
if (!fs.exists(path)) \{  
fs.mkdirs(path);  
\}  
String filename = null;  
for (int i = 1; i <= 4; i++) \{  
filename = "counties\_" + i + ".csv";  
localPath = new Path("counties/" + filename);  
destPath = new Path("counties/" + filename);  
fs.copyFromLocalFile(localPath, destPath);  
\}

**build.gradle:**

project.ext.mainclass = 'hdfs.InputCounties'  
project.ext.archiveName = 'inputcounties.jar'  
apply from: '/root/java/labs/build.gradle'

**yarn jar inputcounties.jar hdfs.InputCounties**

**Demo: Understanding MapReduce**

Why are the words sorted alphabetically?

**The words are the keys, and keys get sorted during the shuffle/sort phase.**

**Lab: Word Count**

**WordCountMapper**

package wordcount;  
  
import java.io.IOException;  
import org.apache.commons.lang.StringUtils;  
import org.apache.hadoop.io.IntWritable;  
import org.apache.hadoop.io.LongWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Mapper;  
  
  
public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> \{  
private static final IntWritable ONE = new IntWritable(1);  
private Text outputKey = new Text();  
@Override  
protected void map(LongWritable key, Text value,  
Mapper<LongWritable, Text, Text, IntWritable>.Context context)  
throws IOException, InterruptedException \{  
String lineStr = value.toString();  
String\[\] words = StringUtils.split(lineStr, ' ');  
//super.map(key, value, context);  
for (String word : words) \{  
outputKey.set(word);  
context.write(outputKey, ONE);  
\}  
\}  
@Override  
protected void setup(  
Mapper<LongWritable, Text, Text, IntWritable>.Context context)  
throws IOException, InterruptedException \{  
// TODO Auto-generated method stub  
super.setup(context);  
\}  
  
@Override  
protected void cleanup(  
Mapper<LongWritable, Text, Text, IntWritable>.Context context)  
throws IOException, InterruptedException \{  
// TODO Auto-generated method stub  
super.cleanup(context);  
\}  
  
\}

WordCountReducer

package wordcount;  
  
import java.io.IOException;  
import org.apache.hadoop.io.IntWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Reducer;  
  
public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> \{  
private IntWritable outputValue = new IntWritable();  
@Override  
protected void reduce(Text key, Iterable<IntWritable> values,  
Reducer<Text, IntWritable, Text, IntWritable>.Context context)  
throws IOException, InterruptedException \{  
int sum = 0;  
for (IntWritable value : values) \{  
sum += value.get();  
\}  
outputValue.set(sum);  
context.write(key, outputValue);  
\}  
  
\}

**WordCountJob**

package wordcount;  
  
import org.apache.hadoop.conf.Configuration;  
import org.apache.hadoop.conf.Configured;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.IntWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;  
import org.apache.hadoop.util.Tool;  
import org.apache.hadoop.util.ToolRunner;  
  
public class WordCountJob extends Configured implements Tool \{  
  
public static void main(String\[\] args) throws Exception \{  
int result = ToolRunner.run(new Configuration(), new WordCountJob(), args);  
System.exit(result);  
\}  
  
@Override  
public int run(String\[\] args) throws Exception \{  
Job job = Job.getInstance(getConf(), "WordCountJob");  
Configuration configuration = job.getConfiguration();  
job.setJarByClass(getClass());  
Path in = new Path(args\[0\]);  
Path out = new Path(args\[1\]);  
out.getFileSystem(configuration).delete(out, true);  
FileInputFormat.setInputPaths(job, in);  
FileOutputFormat.setOutputPath(job, out);  
job.setMapperClass(WordCountMapper.class);  
job.setReducerClass(WordCountReducer.class);  
job.setInputFormatClass(TextInputFormat.class);  
job.setOutputFormatClass(TextOutputFormat.class);  
job.setMapOutputKeyClass(Text.class);  
job.setMapOutputValueClass(IntWritable.class);  
job.setOutputKeyClass(Text.class);  
job.setOutputValueClass(IntWritable.class);  
return job.waitForCompletion(true) ? 0 : 1;  
\}  
\}