Spark 计算人员三度关系

1、一度人脉：双方直接是好友
 
2、二度人脉：双方有一个以上共同的好友，这时朋友网可以计算出你们有几个共同的好友并且呈现数字给你。你们的关系是： 你->朋友->陌生人
 
3、三度人脉：即你朋友的朋友的朋友就是这个陌生人。你们的关系是 你->朋友->朋友->陌生人
 
4、四度人脉：比三度增加一度，你们的关系是，你->朋友->朋友->朋友->陌生人
 
5、五度人脉：你->朋友->朋友->朋友->朋友->陌生人 ，像上面这张图片表示的就是一个五度人脉关系。
 
6、六度人脉：你->朋友->朋友->朋友->朋友->朋友->陌生人

数据格式如下：

A,B  
A,C  
A,E  
B,D  
E,D  
C,F  
F,G  

业务逻辑如下：

1、转化操作flatMapToPair将行数据变为键值对，如A，B表示A和B认识，A可以通过B认识B的朋友，B通过A可以认识A的朋友，转化结果为｛A：A，B，deg1friend，A->B｝、｛B：B，A，deg1friend，B->A｝；

2、转化操作groupByKey对键值对按Key进行分组，转化结果为：｛A，【A，B ，deg1friend，A->B，A，E ，deg1friend，A->E， A，C，deg1friend，A->C 】｝...；

3、转化操作flatMapToPair生成包含可能存在（A->B,A->C两者走向B和C不相同，但都认识A，B和C即存在可能）二度关系的新的键值对，如A和B认识且A与C认识，那么B与C可以存在认识关系即二度关系，路线走向为：B->A->C或C->A->B；

4、转化操作filter在新的键值对中筛选出一度关系即两者已经是认识的，如A和B认识是一度关系；

5、转化操作subtractByKey对包含二度关系的键值对删除存在一度关系的人员及只剩二度关系；

6、转化操作flatMapToPair生成新的二度关系及走向（双向走向【B，C，deg2friend，C->A->B，B，C，deg2friend，B->A->C】）；

7，将新的二度关系与一度关系进行合并；

8、转化操作groupByKey对键值对按Key进行分组，转化结果为：（B，【B，A，deg1friend，B->A， B，D，deg1friend，B->D， B，C，deg2friend，C->A->B， B，E，deg2friend，B->A->E， B，E，deg2friend，B->D->E， B，E，deg2friend，E->A->B， B，E，deg2friend，E->D->B， B，C，deg2friend，B->A->C 】）...；

9、转化操作flatMapToPair生成包含可能存在（如：B->C，deg2friend，C->A->B  ，  B->D，deg1friend，B->D ，判断条件前为deg2friend，后为 deg1friend，前split【0】= 后split【0】，后的起点不在前的路径内）三度关系的新的键值对；

10、转化操作subtractByKey对包含三度关系的键值对删除存在一度关系的人员；

11、行为操作countByKey统计存在三度关系的比重；

具有实现：

package com.test;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.storage.StorageLevel;

import scala.Tuple2;

public class Test2 {

	public static void main(String[] args) {
		SparkConf conf = new SparkConf().setMaster("local").setAppName("My Test APP");
		
		JavaSparkContext sc = new JavaSparkContext(conf);
		
		JavaRDD<String> rdd = sc.textFile("C:/rmgx.txt");
		
		JavaPairRDD<String, String> r1 = rdd.flatMapToPair(new PairFlatMapFunction<String,String,String>(){
			@Override
			public Iterator<Tuple2<String, String>> call(String t)
					throws Exception {
				List<Tuple2<String, String>> list = new ArrayList(); 
				String[] eachterm = t.split(",");
				list.add(new Tuple2(eachterm[0], eachterm[0] + "," + eachterm[1] + "," + "deg1friend"+ "," + eachterm[0] + "->" + eachterm[1]));
				list.add(new Tuple2(eachterm[1], eachterm[1] + "," + eachterm[0] + "," + "deg1friend"+ "," + eachterm[1] + "->" + eachterm[0]));
				return list.iterator();
			}
			
		});
		
		r1.persist(StorageLevel.DISK_ONLY());
						
		JavaPairRDD<String, Iterable<String>> r2 = r1.groupByKey();
					
		JavaPairRDD<String, String> r3 = r2.flatMapToPair(new PairFlatMapFunction<Tuple2<String,Iterable<String>>,String,String>(){

			@Override
			public Iterator<Tuple2<String, String>> call(
					Tuple2<String, Iterable<String>> t) throws Exception {
				List<Tuple2<String, String>> list = new ArrayList(); 
				for (Iterator iter = t._2.iterator(); iter.hasNext();) {
				     String str1 = (String)iter.next();
				     String str1_0 = str1.split(",")[0];
				     String str1_1 = str1.split(",")[1];
				     list.add(new Tuple2(str1_0+ "->" + str1_1,"deg1friend,"+str1_0+ "->" + str1_1));
				     for (Iterator iter2 = t._2.iterator(); iter2.hasNext();) {
				    	 String str2 = (String)iter2.next();
				    	 String str2_0 = str2.split(",")[0];
				    	 String str2_1 = str2.split(",")[1];  
					     if(!str1_1.equals(str2_1)){
					    	 list.add(new Tuple2(str1_1+ "->" + str2_1 ,"deg2friend,"+str1_1 + "->" + str2_0 + "->" + str2_1)); 
					     }
				     }
				}
				return list.iterator();
			}
		});
		
		JavaPairRDD<String, String> r4 = r3.filter(new Function<Tuple2<String,String>,Boolean>(){
			@Override
			public Boolean call(Tuple2<String, String> v1) throws Exception {
				return v1._2.indexOf("deg1friend")>-1;
			}	
		});
		
		r4.persist(StorageLevel.DISK_ONLY());
		
		JavaPairRDD<String, String> r5 = r3.subtractByKey(r4);
		
		JavaPairRDD<String, String> r6 = r5.flatMapToPair(new PairFlatMapFunction<Tuple2<String,String>,String,String>(){
			@Override
			public Iterator<Tuple2<String, String>> call(
					Tuple2<String, String> t) throws Exception {
				List<Tuple2<String, String>> list = new ArrayList(); 
				String t0 = t._1.split("->")[0];
				String t1 = t._1.split("->")[1];
				String t2_1 = t._2.split(",")[1];
				list.add(new Tuple2(t0, t0 + "," + t1 + "," + "deg2friend"+ "," +t2_1));
				list.add(new Tuple2(t1, t1 + "," + t0 + "," + "deg2friend"+ "," +t2_1));
				return list.iterator();
			}
		});
		
		JavaPairRDD<String, String> r7= r1.union(r6);
		
		JavaPairRDD<String, Iterable<String>> r8 = r7.groupByKey();
		
		System.out.println("线路走向："+StringUtils.join(r8.collect(), ","));
		
		JavaPairRDD<String, String> r9 = r8.flatMapToPair(new PairFlatMapFunction<Tuple2<String,Iterable<String>>,String,String>(){
			@Override
			public Iterator<Tuple2<String, String>> call(
					Tuple2<String, Iterable<String>> t) throws Exception {
				List<Tuple2<String, String>> list = new ArrayList(); 
				for (Iterator iter = t._2.iterator(); iter.hasNext();) {
				     String str1 = (String)iter.next();
				     String str1_0 = str1.split(",")[0];
				     String str1_1 = str1.split(",")[1];
				     String str1_2 = str1.split(",")[2];
				     String str1_3 = str1.split(",")[3];
				     for (Iterator iter2 = t._2.iterator(); iter2.hasNext();) {
				    	 String str2 = (String)iter2.next();
				    	 String str2_0 = str2.split(",")[0];
					     String str2_1 = str2.split(",")[1];
					     String str2_2 = str2.split(",")[2];
					     String str2_3 = str2.split(",")[3];
					     if(!str1_1.equals(str2_1) && str1_2.equals("deg2friend") && str2_2.equals("deg1friend") && !(str1_3.indexOf(str2_1)>-1) && (str1_3.split("->")[0].equals(str1_1))
					    		 &&str1_0.equals(str2_0)) {
					    	 list.add(new Tuple2(str1_1+ "->" + str2_1 ,"deg3friend,"+str1_3+"->"+str2_1)); 
					     }
				     }
				}
				return list.iterator();
			}
		});
		JavaPairRDD<String, String> r10 = r9.subtractByKey(r4);
		
		System.out.println("线路走向："+StringUtils.join(r10.collect(), ","));
		
		Map<String, Long> r11 = r10.countByKey();
		
		System.out.println(r11);
		
	}
}

运行结果如下：

{C->D=2, B->F=1, G->A=1, F->E=1, F->B=1, E->F=1, D->C=2, A->G=1}