Hadoop_MapReduce_求最大值和最小值

拼搏现实的明天。 2023-03-13 07:00 23阅读 0赞

**Hadoop\_MapReduce\_求最大值和最小值**

1.  原始数据：D:\\data\\nums.txt  
    需求：找出最大和最小的数字

3
    4
    1
    10
    15
    22
    8
    hadoop
    5K
    62
    1

1.  MaxMin

package com.blu.maxmin;
    
    import java.io.DataInput;
    import java.io.DataOutput;
    import java.io.IOException;
    
    import org.apache.hadoop.io.WritableComparable;
    
    public class MaxMin implements WritableComparable<MaxMin>{ 
    	
    	private int max;
    	private int min;
    	private int total;
    
    	public void readFields(DataInput in) throws IOException { 
    		max = in.readInt();
    		min = in.readInt();
    		total = in.readInt();
    	}
    
    	public void write(DataOutput out) throws IOException { 
    		out.writeInt(max);
    		out.writeInt(min);
    		out.writeInt(total);
    	}
    
    	public int compareTo(MaxMin o) { 
    		return 0;
    	}
    
    	public int getMax() { 
    		return max;
    	}
    
    	public void setMax(int max) { 
    		this.max = max;
    	}
    
    	public int getMin() { 
    		return min;
    	}
    
    	public void setMin(int min) { 
    		this.min = min;
    	}
    
    	public int getTotal() { 
    		return total;
    	}
    
    	public void setTotal(int total) { 
    		this.total = total;
    	}
    
    	@Override
    	public String toString() { 
    		return "MaxMin [max=" + max + ", min=" + min + ", total=" + total + "]";
    	}
    	
    	public MaxMin() { 
    		super();
    	}
    
    	public MaxMin(int max, int min, int total) { 
    		super();
    		this.max = max;
    		this.min = min;
    		this.total = total;
    	}
    	
    }

1.  MaxMinMapper

package com.blu.maxmin;
    
    import java.io.IOException;
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.NullWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    public class MaxMinMapper extends Mapper<LongWritable, Text, MaxMin, NullWritable>{ 
    	
    	private int max;
    	private int min;
    	private int total = 0;
    	
    	@Override
    	protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, MaxMin, NullWritable>.Context context)
    			throws IOException, InterruptedException { 
    		String val = value.toString();
    		try { 
    			int current = Integer.parseInt(val);
    			if(total == 0) { 
    				min = current;
    				max = current;
    			}
    			if(current>max) { 
    				max = current;
    			}
    			if(current<min) { 
    				min = current;
    			}
    		} catch (Exception e) { 
    			e.printStackTrace();
    			return;
    		}
    		total++;
    	}
    	
    	/** * cheanup方法会在所有的map方法执行完成后执行，只会执行一次。 */
    	@Override
    	protected void cleanup(Mapper<LongWritable, Text, MaxMin, NullWritable>.Context context)
    			throws IOException, InterruptedException { 
    		MaxMin mm = new MaxMin(max,min,total);
    		context.write(mm, NullWritable.get());
    	}
    }

1.  MaxMinReduce

package com.blu.maxmin;
    
    import java.io.IOException;
    
    import org.apache.hadoop.io.NullWritable;
    import org.apache.hadoop.mapreduce.Reducer;
    
    public class MaxMinReduce extends Reducer<MaxMin, NullWritable, MaxMin, NullWritable>{ 
    	@Override
    	protected void reduce(MaxMin key, Iterable<NullWritable> value,
    			Reducer<MaxMin, NullWritable, MaxMin, NullWritable>.Context context) throws IOException, InterruptedException { 
    		context.write(key, NullWritable.get());
    	}
    }

1.  MaxMinJob

package com.blu.maxmin;
    
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.NullWritable;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    public class MaxMinJob { 
    	public static void main(String[] args) throws Exception { 
    		Job job = Job.getInstance();
    		job.setJarByClass(MaxMinJob.class);
    		job.setMapperClass(MaxMinMapper.class);
    		job.setReducerClass(MaxMinReduce.class);
    		job.setMapOutputKeyClass(MaxMin.class);
    		job.setMapOutputValueClass(NullWritable.class);
    		job.setOutputKeyClass(MaxMin.class);
    		job.setOutputValueClass(NullWritable.class);
    		FileInputFormat.addInputPath(job, new Path(args[0]));
    		FileOutputFormat.setOutputPath(job, new Path(args[1]));
    		boolean flag = job.waitForCompletion(true);
    		System.exit(flag ? 0 : 1);
    	}
    }

1.  运行参数：

D:\data\nums.txt D:\data\output

1.  运行结果：

MaxMin [max=62, min=1, total=9]