安徽省大数据网络赛大数据分析第一小题

向右看齐 2023-08-17 17:47 100阅读 0赞

该数据集log.log是某APP用户日志，请你使用MapReduce程序对数据进

行预处理清洗提取。（15分）

1.1 原始数据预处理（编写相关代码及部分结果截图8分）

*规则一*：如果没有数据中没有uid、platform、app\_version、pid四个字段同时出现的数据，请过滤掉

*规则二*：将数据中字段locationcity的值为0的全部替换为1

**数据类型：**

### 解题思路： ###

**首先我们观察数据的格式，按照什么切分，最终确定按照逗号切分效果最好。**

**第一步：map阶段进行过滤，只要包含这四个字段就进行写进reduce,并且把包含locationcity的0直接替换为1.**

**核心算法我提取出来：来了一个value我们需要进行字符串转换，并且切割，然后对应的字段的位置是否包含我们想要的数据。**

String line = value.toString();
    //{"common":{"locationcity":0,"uid":"188495963831271424","uaid":"0","platform":"Android","app_version":"1007090002","net":"WIFI","pid":"5057","identifier":"869121033612809","cityid":"2503","iccid":"89860077221897301901","snsid":"","ts":"1557276436920","versionType":"1","pkg":"com.moji.mjweather"}
    		//,"event":{"key":"NEWLIVEVIEW_QUIT_TAB","value":"0","du":""}}
    		String split[] = line.split(",");
    		if(split[1].contains("uid")&&split[3].contains("platform")&&split[4].contains("app_version")&&split[6].contains("pid")){
    			//{"common":{"locationcity":0当成value
    			//其余为key
    			//把第一个字段中有0的替换为1；
    			String convert = split[0];
    			split[0]=convert.replaceAll("0","1");
    					
    			String val = line.substring(split[0].length(),line.length());
    			String keys = split[0];
    			context.write(new Text(keys),new Text(val));
    		}

主函数：
    package jinsai;
    import java.io.IOException;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    public class MapRedu {
    	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
    Configuration conf = new Configuration();
    		Job job = Job.getInstance(conf,MapRedu.class.getSimpleName());		
    		job.setJarByClass(MapRedu.class);
    		FileInputFormat.setInputPaths(job, new Path(args[0]));		
    		job.setMapperClass(MAp.class);
    		job.setMapOutputKeyClass(Text.class);
    		job.setMapOutputValueClass(Text.class);
    
    		job.setReducerClass(Red.class);
    		job.setOutputKeyClass(Text.class);
    		job.setOutputValueClass(Text.class);
    		FileOutputFormat.setOutputPath(job, new Path(args[1]));
    		job.waitForCompletion(true);
    	}
    }
    Map函数中：
    package jinsai;
    
    import java.io.IOException;
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    
    public class MAp extends Mapper<LongWritable, Text, Text, Text>{
    	@Override
    	protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
    			throws IOException, InterruptedException {
    		//对进来的数据进行切分  hello sxt you very good  2
    		String line = value.toString();
    //{"common":{"locationcity":0,"uid":"188495963831271424","uaid":"0","platform":"Android","app_version":"1007090002","net":"WIFI","pid":"5057","identifier":"869121033612809","cityid":"2503","iccid":"89860077221897301901","snsid":"","ts":"1557276436920","versionType":"1","pkg":"com.moji.mjweather"}
    		//,"event":{"key":"NEWLIVEVIEW_QUIT_TAB","value":"0","du":""}}
    		String split[] = line.split(",");
    		if(split[1].contains("uid")&&split[3].contains("platform")&&split[4].contains("app_version")&&split[6].contains("pid")){
    			//{"common":{"locationcity":0当成value
    			//其余为key
    			//把第一个字段中有0的替换为1；
    			String convert = split[0];
    			split[0]=convert.replaceAll("0","1");
    					
    			String val = line.substring(split[0].length(),line.length());
    			String keys = split[0];
    			context.write(new Text(keys),new Text(val));
    		}
    	
    	}
    
    }Reduce函数：
    package jinsai;
    import java.io.IOException;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    public class Red extends Reducer<Text, Text, Text, Text>{
    	@Override
    	protected void reduce(Text key, Iterable<Text> values,
    			Reducer<Text, Text, Text, Text>.Context context) throws IOException, InterruptedException {
    		for (Text text : values) {
    			context.write(new Text(key),new Text(text));
    		}
    	}
    }

**部分数据：**

**结果：**

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQxNDc5NDY0_size_16_color_FFFFFF_t_70][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQxNDc5NDY0_size_16_color_FFFFFF_t_70]: /images/20230808/7712835194af4bfc9c8f71bb77893199.png