Hadoop——测试代码

àì夳堔傛蜴生んèń 2022-05-26 02:00 142阅读 0赞

# 1、本地测试代码 #

class WCMap extends Mapper<LongWritable, Text, Text, IntWritable>\{  
/\*\*  
 \* key: 偏移量  
 \* value:每行文本内容  
 \* context：输出内容上下文  
 \*/  
@Override  
protected void map(LongWritable key, Text value,Context context)  
throws IOException, InterruptedException \{  
IntWritable num = new IntWritable(1);  
String str = value.toString();  
String\[\] words = str.split(" ");  
for (String word : words) \{  
context.write(new Text(word), num);  
\}  
\}  
\}  
class WCReduce extends Reducer<Text, IntWritable, Text, IntWritable>\{  
/\*\*  
 \* v1:map输出的key，即词内容  
 \* itr：对应每个词的值集合  
 \* context:输出内容上下文  
 \*/  
@Override  
protected void reduce(Text v1, Iterable<IntWritable> itr,Context context)  
throws IOException, InterruptedException \{  
int sum = 0;  
for (IntWritable i : itr) \{  
sum += i.get();  
\}  
context.write(v1, new IntWritable(sum));  
\}  
\}  
public class WCJob \{  
public static void main(String\[\] args) throws IOException, ClassNotFoundException, InterruptedException \{  
//默认加载src目录下的配置文件（本地测试时，src下不能添加MR的配置文件）  
Configuration conf = new Configuration();  
//本地测试环境  
conf.set("fs.defaultFS", "hdfs://slave1:8020");  
conf.set("yarn.resourcemanager.hostname", "slave3");  
  
Job job = Job.getInstance(conf);//实例化job  
  
job.setJarByClass(WCJob.class);  
  
job.setMapperClass(WCMap.class);  
job.setMapOutputKeyClass(Text.class);  
job.setMapOutputValueClass(IntWritable.class);  
  
job.setReducerClass(WCReduce.class);  
  
Path inPath = new Path("/wc/input/wc.txt");//设置输入文件  
FileInputFormat.addInputPath(job, inPath);  
  
Path outPath = new Path("/wc/output");//设置结果保存路径  
FileSystem fs = FileSystem.get(conf);  
if (fs.exists(outPath)) \{  
fs.delete(outPath,true);  
\}  
FileOutputFormat.setOutputPath(job, outPath);  
  
boolean flag = job.waitForCompletion(true);  
if (flag) \{  
System.out.println("Job Success!");  
\}  
\}  
\}

# 2、服务器测试代码 #

Configuration conf = new Configuration();

Job job = Job.getInstance(conf);//实例化job

job.setJarByClass(WCJob.class);  
  
job.setMapperClass(WCMap.class);  
job.setMapOutputKeyClass(Text.class);  
job.setMapOutputValueClass(IntWritable.class);  
  
job.setReducerClass(WCReduce.class);  
  
Path inPath = new Path("/wc/input/wc.txt");//设置输入文件  
FileInputFormat.addInputPath(job, inPath);  
  
Path outPath = new Path("/wc/output");//设置结果保存路径  
FileSystem fs = FileSystem.get(conf);  
if (fs.exists(outPath)) \{  
fs.delete(outPath,true);  
\}  
FileOutputFormat.setOutputPath(job, outPath);  
  
boolean flag = job.waitForCompletion(true);  
if (flag) \{  
System.out.println("Job Success!");  
\}  
\}  
\}