Hadoop-倒排索引即多Job串行执行

朱雀 2022-10-19 04:24 93阅读 0赞

## 倒排索引（多Job串联） ##

目的：有多个文本，需要建立搜索索引

数据输入：

# hulaoguan.txt
    wei caocao
    wei xiahou
    wei caoren
    shu liubei
    shu guanyu
    shu zhangfei
    wu sunjian
    wu sunce
    wu sunquan
    
    # chibi.txt
    wei caocao
    wei xiahou
    wei caoren
    shu liubei
    shu guanyu
    shu zhangfei
    shu zhuge
    wu sunquan
    wu zhouyu
    
    # yiling.txt
    shu liubei
    wu sunquan
    
    # wuzhangyuan.txt
    shu zhuge
    wei sima

期望输出数据：

# 以wei为例
    wei hulaoguan.txt->3 chibi.txt->3 wuzhangyuan.txt->1

第一次Mapper

public class FirstJoinMapper extends Mapper<LongWritable, Text, Text, IntWritable> { 
    
        String filename;
        IntWritable v = new IntWritable(1);
        Text k = new Text();
    
        @Override
        protected void setup (Context context) throws IOException, InterruptedException { 
    
            // 获取文件的名称
            // 获取切片信息，并强转成文件切片
            FileSplit inputSplit = (FileSplit) context.getInputSplit();
            filename = inputSplit.getPath().getName();
        }
    
        @Override
        protected void map (LongWritable key, Text value, Context context) throws IOException, InterruptedException { 
    
            // wei caocao
            String line = value.toString();
    
            String[] fileds = line.split(" ");
    
            for (String filed : fileds) { 
    
                k.set(filed + "--" + filename);
                context.write(k, v);
            }
        }
    }

第一次Reducer

public class FirstJoinReducer extends Reducer<Text, IntWritable, Text, IntWritable> { 
    
        IntWritable v = new IntWritable();
    
        @Override
        protected void reduce (Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { 
    
            int sum = 0;
            for (IntWritable value : values) { 
                sum += value.get();
            }
            v.set(sum);
    
            context.write(key, v);
        }
    }

第一次输出结果如下：

caocao--chibi.txt	1
    caocao--hulaoguan.txt	1
    caoren--chibi.txt	1
    caoren--hulaoguan.txt	1
    guanyu--chibi.txt	1
    guanyu--hulaoguan.txt	1
    liubei--chibi.txt	1
    liubei--hulaoguan.txt	1
    liubei--yiling.txt	1
    shu--chibi.txt	4
    shu--hulaoguan.txt	3
    shu--wuzhangyuan.txt	1
    shu--yiling.txt	1
    sima--wuzhangyuan.txt	1
    sunce--hulaoguan.txt	1
    sunjian--hulaoguan.txt	1
    sunquan--chibi.txt	1
    sunquan--hulaoguan.txt	1
    sunquan--yiling.txt	1
    wei--chibi.txt	3
    wei--hulaoguan.txt	3
    wei--wuzhangyuan.txt	1
    wu--chibi.txt	2
    wu--hulaoguan.txt	3
    wu--yiling.txt	1
    xiahou--chibi.txt	1
    xiahou--hulaoguan.txt	1
    zhangfei--chibi.txt	1
    zhangfei--hulaoguan.txt	1
    zhouyu--chibi.txt	1
    zhuge--chibi.txt	1
    zhuge--wuzhangyuan.txt	1

第二次Mapper

public class SecondJoinMapper extends Mapper<LongWritable, Text, Text, Text> { 
    
        Text k = new Text();
        Text v = new Text();
    
        @Override
        protected void map (LongWritable key, Text value, Context context) throws IOException, InterruptedException { 
    
            String line = value.toString();
            String[] fields = line.split("--");
    
            k.set(fields[0]);
            v.set(fields[1]);
    
            context.write(k, v);
        }
    }

第二次Reducer

public class SecondJoinReducer extends Reducer<Text, Text, Text, Text> { 
    
        Text v = new Text();
    
        @Override
        protected void reduce (Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { 
    
            // caocao chibi.txt 1
            // caocao hulaoguan.txt 1
    
            StringBuilder sb = new StringBuilder();
    
            for (Text value : values) { 
                sb.append(value.toString().replace("\t", "-->")).append("\t");
            }
    
            v.set(sb.toString());
    
            context.write(key, v);
        }
    }

第二次运行结果

caocao	hulaoguan.txt-->1	chibi.txt-->1	
    caoren	chibi.txt-->1	hulaoguan.txt-->1	
    guanyu	chibi.txt-->1	hulaoguan.txt-->1	
    liubei	hulaoguan.txt-->1	chibi.txt-->1	yiling.txt-->1	
    shu	chibi.txt-->4	hulaoguan.txt-->3	wuzhangyuan.txt-->1	yiling.txt-->1	
    sima	wuzhangyuan.txt-->1	
    sunce	hulaoguan.txt-->1	
    sunjian	hulaoguan.txt-->1	
    sunquan	hulaoguan.txt-->1	yiling.txt-->1	chibi.txt-->1	
    wei	chibi.txt-->3	hulaoguan.txt-->3	wuzhangyuan.txt-->1	
    wu	yiling.txt-->1	chibi.txt-->2	hulaoguan.txt-->3	
    xiahou	chibi.txt-->1	hulaoguan.txt-->1	
    zhangfei	chibi.txt-->1	hulaoguan.txt-->1	
    zhouyu	chibi.txt-->1	
    zhuge	chibi.txt-->1	wuzhangyuan.txt-->1

##  ##