UserView--第二种方式（避免第一种方式Set饱和），基于Spark算子的java代码实现

 

---

UserView--第二种方式（避免第一种方式Set饱和），基于Spark算子的java代码实现

 

测试数据

java代码

package com.hzf.spark.study;

import java.util.Map;
import java.util.Set;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.broadcast.Broadcast;

import scala.Tuple2;

public class UVAnalysis02 {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName("UV_ANA").setMaster("local")
                .set("spark.testing.memory", "2147480000");
        @SuppressWarnings("resource")
        JavaSparkContext sc = new JavaSparkContext(conf);
        JavaRDD<String> logRDD = sc.textFile("userLog1");
        String str = "View";
        final Broadcast<String> broadcast = sc.broadcast(str);
        uvAnalyzeOptz(logRDD, broadcast);
    }
    
    private static void uvAnalyzeOptz(JavaRDD<String> logRDD, final Broadcast<String> broadcast) {
        JavaRDD<String> filteredLogRDD = logRDD.filter(new Function<String, Boolean>() {
          
            private static final long serialVersionUID = 1L;

            @Override
            public Boolean call(String v1) throws Exception {
                String actionParam = broadcast.value();
                String action = v1.split("	")[5];
                return actionParam.equals(action);
            }
        });
        
        JavaPairRDD<String, String> up2LogRDD = filteredLogRDD.mapToPair(new PairFunction<String, String, String>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Tuple2<String, String> call(String val) throws Exception {
                String[] splited = val.split("	");
                String userId = splited[2];
                String pageId = splited[3];
                 
                return new Tuple2<String, String>(userId + "_" + pageId,null);
            }
        });
        
        JavaPairRDD<String, Iterable<String>> groupUp2LogRDD = up2LogRDD.groupByKey();
        
        Map<String, Object> countByKey = groupUp2LogRDD.mapToPair(new PairFunction<Tuple2<String,Iterable<String>>, String, String>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Tuple2<String, String> call(Tuple2<String, Iterable<String>> tuple) throws Exception {
                String pu = tuple._1;
                String[] splited = pu.split("_");
                String pageId = splited[1];
                return new Tuple2<String, String>(pageId,null);
            }
        }).countByKey();
        
        Set<String> keySet = countByKey.keySet();
        for (String key : keySet) {
            System.out.println("PAGEID:"+key+"	UV_COUNT："+countByKey.get(key));
        }
    }
}

result