mapreduce程序案例4--查找QQ共同好友

秒速五厘米 2022-12-13 12:48 150阅读 0赞

需求：

查找QQ共同好友：

A:B,C,D,F,E,O
    B:A,C,E,K
    C:F,A,D,I
    D:A,E,F,L
    E:B,C,D,M,L
    F:A,B,C,D,E,O,M
    G:A,C,D,E,F
    H:A,C,D,E,O
    I:A,O
    J:B,O
    K:A,C,D
    L:D,E,F
    M:E,F,G
    O:A,H,I,J

思路：

找出key为B,C,D,F,E,O的共同好友为A，然后reduce，去重，获取所有QQ的共同好友列表；

package application.mapreduce;
    
    import lombok.AllArgsConstructor;
    import lombok.Builder;
    import lombok.Data;
    import lombok.NoArgsConstructor;
    import org.apache.hadoop.io.WritableComparable;
    
    import java.io.DataInput;
    import java.io.DataOutput;
    import java.io.IOException;
    import java.util.Arrays;
    import java.util.Set;
    import java.util.TreeSet;
    
    /**
     * @author: wtl
     * @License: (C) Copyright 2020, wtl Corporation Limited.
     * @Contact: 1050100468@qq.com
     * @Date: 2020-10-11 18:16
     * @Version: 1.0
     * @Description:
     */
    @Data
    @AllArgsConstructor
    @NoArgsConstructor
    @Builder
    public class Friend implements WritableComparable<Friend> {
        private String friend;
        @Override
        public int compareTo(Friend o) {
            String[] fs = friend.split("_");
            String[] f2s = o.friend.split("_");
    
            Set<String> fSet = new TreeSet<>(Arrays.asList(fs));
            Set<String> f2Set = new TreeSet<>(Arrays.asList(f2s));
    
            StringBuilder stringBuilderF1 = new StringBuilder();
            StringBuilder stringBuilderF2 = new StringBuilder();
    
            fSet.forEach(stringBuilderF1::append);
            f2Set.forEach(stringBuilderF2::append);
            return stringBuilderF1.toString().compareTo(stringBuilderF2.toString());
        }
    
        @Override
        public void write(DataOutput dataOutput) throws IOException {
            dataOutput.writeUTF(friend);
        }
    
        @Override
        public void readFields(DataInput dataInput) throws IOException {
            friend = dataInput.readUTF();
        }
    }

package application.mapreduce;
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    import java.io.IOException;
    
    /**
     * @author: wtl
     * @License: (C) Copyright 2020, wtl Corporation Limited.
     * @Contact: 1050100468@qq.com
     * @Date: 2020-10-11 18:15
     * @Version: 1.0
     * @Description:
     */
    public class FriendsMapper extends Mapper<LongWritable, Text,Friend,Text> {
    
        private Text text = new Text();
    
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            String[] fields = line.split(":");
    
            String friend = fields[0];
    
            String[] persons = fields[1].split(",");
    
            text.set(friend);
    
            for (int i = 0; i < persons.length; i++) {
                for (int j=i+1;j< persons.length;j++){
                    Friend build = Friend.builder()
                            .friend(persons[i] + "_" + persons[j])
                            .build();
                    context.write(build,text);
                }
            }
        }
    }

package application.mapreduce;
    
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    import java.io.IOException;
    import java.util.Set;
    import java.util.TreeSet;
    
    /**
     * @author: wtl
     * @License: (C) Copyright 2020, wtl Corporation Limited.
     * @Contact: 1050100468@qq.com
     * @Date: 2020-10-11 18:36
     * @Version: 1.0
     * @Description:
     */
    public class FriendsReducer extends Reducer<Friend, Text,Friend,Text> {
    
        private Text text = new Text();
    
        @Override
        protected void reduce(Friend key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            Set<String> set = new TreeSet<>();
    
            for (Text value : values){
                set.add(value.toString());
            }
            StringBuilder stringBuilder = new StringBuilder();
            set.forEach(s -> {
                stringBuilder.append(s).append(",");
            });
            text.set(stringBuilder.substring(0,stringBuilder.length() - 1));
            context.write(key,text);
        }
    }

输出结果：

Friend(friend=A_B)	F
    Friend(friend=A_C)	B,F,G,H,K
    Friend(friend=A_D)	C,F,G,H,K
    Friend(friend=A_E)	B,D,F,G,H
    Friend(friend=A_F)	C,D,G
    Friend(friend=A_H)	O
    Friend(friend=A_I)	C,O
    Friend(friend=A_J)	O
    Friend(friend=A_K)	B
    Friend(friend=A_L)	D
    Friend(friend=A_M)	F
    Friend(friend=A_O)	F,H,I
    Friend(friend=B_C)	A,E,F
    Friend(friend=B_D)	A,E,F
    Friend(friend=B_E)	A,F
    Friend(friend=B_F)	A
    Friend(friend=B_L)	E
    Friend(friend=B_M)	E,F
    Friend(friend=B_O)	A,F,J
    Friend(friend=C_D)	A,E,F,G,H,K
    Friend(friend=C_E)	A,B,F,G,H
    Friend(friend=C_F)	A,G
    Friend(friend=C_K)	B
    Friend(friend=C_L)	E
    Friend(friend=C_M)	E,F
    Friend(friend=C_O)	A,F,H
    Friend(friend=D_E)	A,F,G,H,L
    Friend(friend=D_F)	A,C,G,L
    Friend(friend=D_I)	C
    Friend(friend=D_L)	E
    Friend(friend=D_M)	E,F
    Friend(friend=D_O)	A,F,H
    Friend(friend=F_E)	A,D,G,L,M
    Friend(friend=E_G)	M
    Friend(friend=E_K)	B
    Friend(friend=E_L)	D
    Friend(friend=E_M)	F
    Friend(friend=E_O)	A,F,H
    Friend(friend=F_G)	M
    Friend(friend=F_I)	C
    Friend(friend=F_L)	D
    Friend(friend=F_O)	A
    Friend(friend=H_I)	O
    Friend(friend=H_J)	O
    Friend(friend=I_J)	O
    Friend(friend=M_L)	E
    Friend(friend=O_M)	F