10、spark高级编程

一、基于排序机制的wordcount程序

1、要求

1、对文本文件内的每个单词都统计出其出现的次数。

2、按照每个单词出现次数的数量，降序排序。

2、代码实现

------java实现-------

package cn.spark.study.core;

import java.util.Arrays;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;

import scala.Tuple2;

public class SortWordCount {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName("SortWordCount").setMaster("local");
        JavaSparkContext sc = new JavaSparkContext(conf);
        
        JavaRDD<String> lines = sc.textFile("D:\test-file\spark.txt");
        
        JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Iterable<String> call(String t) throws Exception {
                return Arrays.asList(t.split(" "));
            }
        });
        
        JavaPairRDD<String, Integer> pairs = words.mapToPair(new PairFunction<String, String, Integer>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Tuple2<String, Integer> call(String t) throws Exception {
                return new Tuple2<String, Integer>(t, 1);
            }
        });
        
        JavaPairRDD<String, Integer> wordCounts = pairs.reduceByKey(new Function2<Integer, Integer, Integer>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Integer call(Integer v1, Integer v2) throws Exception {
                return v1 + v2;
            }
        });
        
        // 到这里为止，就得到了每个单词出现的次数
        // 但是，问题是，我们的新需求，是要按照每个单词出现次数的顺序，降序排序
        // wordCounts RDD内的元素是什么？应该是这种格式的吧：(hello, 3) (you, 2)
        // 我们需要将RDD转换成(3, hello) (2, you)的这种格式，才能根据单词出现次数进行排序把！
 
        // 进行key-value的反转映射
        JavaPairRDD<Integer, String> countWords = wordCounts.mapToPair(new PairFunction<Tuple2<String,Integer>, Integer, String>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Tuple2<Integer, String> call(Tuple2<String, Integer> t) throws Exception {
                return new Tuple2<Integer, String>(t._2, t._1);
            }
        });
        
        //按照key进行排序
        JavaPairRDD<Integer, String> sortedCountWords = countWords.sortByKey(false);
        
        //再次将value-key进行反转映射
        JavaPairRDD<String, Integer> sortedWordCounts = sortedCountWords.mapToPair(new PairFunction<Tuple2<Integer,String>, String, Integer>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Tuple2<String, Integer> call(Tuple2<Integer, String> t) throws Exception {
                return new Tuple2<String, Integer>(t._2, t._1);
            }
        });
        

        // 到此为止，我们获得了按照单词出现次数排序后的单词计数
        // 打印出来
        sortedWordCounts.foreach(new VoidFunction<Tuple2<String,Integer>>() {

            private static final long serialVersionUID = 1L;

            @Override
            public void call(Tuple2<String, Integer> t) throws Exception {
                System.out.println(t._1 + " appears " + t._2 + " times.");
            }
        });
        
        sc.close();
    }

}




---------scala实现---------

package cn.spark.study.core

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext

/**
 * @author Administrator
 */
object SortWordCount {
  
  def main(args: Array[String]) {
    val conf = new SparkConf()
        .setAppName("SortWordCount")
        .setMaster("local") 
    val sc = new SparkContext(conf)
    
    val lines = sc.textFile("D:\test-file\spark.txt", 1)
    val words = lines.flatMap { line => line.split(" ") }  
    val pairs = words.map { word => (word, 1) }  
    val wordCounts = pairs.reduceByKey(_ + _)  
    
    val countWords = wordCounts.map(wordCount => (wordCount._2, wordCount._1))   
    val sortedCountWords = countWords.sortByKey(false)  
    val sortedWordCounts = sortedCountWords.map(sortedCountWord => (sortedCountWord._2, sortedCountWord._1))  
    
    sortedWordCounts.foreach(sortedWordCount => println(
        sortedWordCount._1 + " appear " + sortedWordCount._2 + " times."))
  }
  
}

二、二次排序

1、要求

1、按照文件中的第一列排序。

2、如果第一列相同，则按照第二列排序。

2、java代码

###SecondarySortKey

package cn.spark.study.core;

import java.io.Serializable;

import scala.math.Ordered;

/**
 * 自定义的二次排序key
 * @author Administrator
 *
 */
public class SecondarySortKey implements Ordered<SecondarySortKey>, Serializable {

    private static final long serialVersionUID = -2366006422945129991L;
    
    // 首先在自定义key里面，定义需要进行排序的列
    private int first;
    private int second;
    
    public SecondarySortKey(int first, int second) {
        this.first = first;
        this.second = second;
    }

    @Override
    public boolean $greater(SecondarySortKey other) {
        if(this.first > other.getFirst()) {
            return true;
        } else if(this.first == other.getFirst() && 
                this.second > other.getSecond()) {
            return true;
        }
        return false;
    }
    
    @Override
    public boolean $greater$eq(SecondarySortKey other) {
        if(this.$greater(other)) {
            return true;
        } else if(this.first == other.getFirst() && 
                this.second == other.getSecond()) {
            return true;
        }
        return false;
    }

    @Override
    public boolean $less(SecondarySortKey other) {
        if(this.first < other.getFirst()) {
            return true;
        } else if(this.first == other.getFirst() && 
                this.second < other.getSecond()) {
            return true;
        }
        return false;
    }
    
    @Override
    public boolean $less$eq(SecondarySortKey other) {
        if(this.$less(other)) {
            return true;
        } else if(this.first == other.getFirst() && 
                this.second == other.getSecond()) {
            return true;
        }
        return false;
    }
    
    @Override
    public int compare(SecondarySortKey other) {
        if(this.first - other.getFirst() != 0) {
            return this.first - other.getFirst();
        } else {
            return this.second - other.getSecond();
        }
    }
    
    @Override
    public int compareTo(SecondarySortKey other) {
        if(this.first - other.getFirst() != 0) {
            return this.first - other.getFirst();
        } else {
            return this.second - other.getSecond();
        }
    }
    
    // 为要进行排序的多个列，提供getter和setter方法，以及hashcode和equals方法
    public int getFirst() {
        return first;
    }

    public void setFirst(int first) {
        this.first = first;
    }

    public int getSecond() {
        return second;
    }

    public void setSecond(int second) {
        this.second = second;
    }

    @Override
    public int hashCode() {
        final int prime = 31;
        int result = 1;
        result = prime * result + first;
        result = prime * result + second;
        return result;
    }

    @Override
    public boolean equals(Object obj) {
        if (this == obj)
            return true;
        if (obj == null)
            return false;
        if (getClass() != obj.getClass())
            return false;
        SecondarySortKey other = (SecondarySortKey) obj;
        if (first != other.first)
            return false;
        if (second != other.second)
            return false;
        return true;
    }
    
}






###SecondarySort

package cn.spark.study.core;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;

import scala.Tuple2;

/**
 * 二次排序
 * 1、实现自定义的key，要实现Ordered接口和Serializable接口，在key中实现自己对多个列的排序算法
 * 2、将包含文本的RDD，映射成key为自定义key，value为文本的JavaPairRDD
 * 3、使用sortByKey算子按照自定义的key进行排序
 * 4、再次映射，剔除自定义的key，只保留文本行
 * @author Administrator
 *
 */
public class SecondarySort {

    public static void main(String[] args) {
        SparkConf conf = new SparkConf()
                .setAppName("SecondarySort") 
                .setMaster("local");
        JavaSparkContext sc = new JavaSparkContext(conf);
    
        JavaRDD<String> lines = sc.textFile("D:\test-file\sort.txt");
        
        JavaPairRDD<SecondarySortKey, String> pairs = lines.mapToPair(
                
                new PairFunction<String, SecondarySortKey, String>() {

                    private static final long serialVersionUID = 1L;

                    @Override
                    public Tuple2<SecondarySortKey, String> call(String line) throws Exception {
                        String[] lineSplited = line.split(" ");  
                        SecondarySortKey key = new SecondarySortKey(
                                Integer.valueOf(lineSplited[0]), 
                                Integer.valueOf(lineSplited[1]));  
                        return new Tuple2<SecondarySortKey, String>(key, line);
                    }
                    
                });
        
        JavaPairRDD<SecondarySortKey, String> sortedPairs = pairs.sortByKey();
        
        JavaRDD<String> sortedLines = sortedPairs.map(
                
                new Function<Tuple2<SecondarySortKey,String>, String>() {

                    private static final long serialVersionUID = 1L;

                    @Override
                    public String call(Tuple2<SecondarySortKey, String> v1) throws Exception {
                        return v1._2;
                    }
                    
                });
        
        sortedLines.foreach(new VoidFunction<String>() {

            private static final long serialVersionUID = 1L;

            @Override
            public void call(String t) throws Exception {
                System.out.println(t);  
            }
            
        });
        
        sc.close();
    }
    
}

3、scala代码

###SecondSortKey

package cn.spark.study.core

/**
 * @author Administrator
 */
class SecondSortKey(val first: Int, val second: Int) 
    extends Ordered[SecondSortKey] with Serializable {
  
  def compare(that: SecondSortKey): Int = {
    if(this.first - that.first != 0) {
      this.first - that.first
    } else {
      this.second - that.second
    }
  }
  
}






###SecondSort

package cn.spark.study.core

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext

/**
 * @author Administrator
 */
object SecondSort {
  
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
        .setAppName("SecondSort")  
        .setMaster("local")  
    val sc = new SparkContext(conf)
  
    val lines = sc.textFile("D:\test-file\sort.txt", 1)
    val pairs = lines.map { line => (
        new SecondSortKey(line.split(" ")(0).toInt, line.split(" ")(1).toInt),
        line)}
    val sortedPairs = pairs.sortByKey()
    val sortedLines = sortedPairs.map(sortedPair => sortedPair._2)  
    
    sortedLines.foreach { sortedLine => println(sortedLine) }  
  }
  
}

三、topn

1、要求

1、对文本文件内的数字，取最大的前3个。

2、对每个班级内的学生成绩，取出前3名。（分组取topn）

3、课后作业：用Scala来实现分组取topn。

2、获取文本内最大的前三个数

---------java实现----------

package cn.spark.study.core;

import java.util.List;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;

import scala.Tuple2;


public class Top3 {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName("Top3Java").setMaster("local");
        JavaSparkContext sc = new JavaSparkContext(conf);
        
        JavaRDD<String> lines = sc.textFile("D:\test-file\top.txt"); 

        JavaPairRDD<Integer, String> pairs = lines.mapToPair(new PairFunction<String, Integer, String>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Tuple2<Integer, String> call(String t) throws Exception {
                return new Tuple2<Integer, String>(Integer.valueOf(t), t);
            }
        });
        
        JavaPairRDD<Integer, String> sortedPairs = pairs.sortByKey(false);
        JavaRDD<Integer> sortedNumbers = sortedPairs.map(new Function<Tuple2<Integer,String>, Integer>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Integer call(Tuple2<Integer, String> v1) throws Exception {
                return v1._1;
            }
        });
        
        List<Integer> sortedNumberList = sortedNumbers.take(3);  //此时sortedNumberList是： [9, 7, 6]
        for(Integer num : sortedNumberList) {
            System.out.println(num);
        }
        
        sc.close();
 
    }
}







---------scala实现----------

package cn.spark.study.core

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext

/**
 * @author Administrator
 */
object Top3 {
  
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
        .setAppName("Top3")
        .setMaster("local")  
    val sc = new SparkContext(conf)
    
    val lines = sc.textFile("D:\test-file\top.txt", 1)
    val pairs = lines.map { line => (line.toInt, line) }
    val sortedPairs = pairs.sortByKey(false)
    val sortedNumbers = sortedPairs.map(sortedPair => sortedPair._1)  
    val top3Number = sortedNumbers.take(3)
    
    for(num <- top3Number) {
      println(num)  
    }
  }
  
}

3、对每个班级内的学生成绩，取出前3名。（分组取topn）

----java实现-----

package cn.spark.study.core;

import java.util.Arrays;
import java.util.Iterator;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;

import scala.Tuple2;

/**
 * 分组取top3
 * @author Administrator
 *
 */
public class GroupTop3 {
    
    public static void main(String[] args) {
        SparkConf conf = new SparkConf()
                .setAppName("Top3")
                .setMaster("local");  
        JavaSparkContext sc = new JavaSparkContext(conf);
        
        JavaRDD<String> lines = sc.textFile("D:\test-file\score.txt");
        
        JavaPairRDD<String, Integer> pairs = lines.mapToPair(
                
                new PairFunction<String, String, Integer>() {

                    private static final long serialVersionUID = 1L;

                    @Override
                    public Tuple2<String, Integer> call(String line) throws Exception {
                        String[] lineSplited = line.split(" ");  
                        return new Tuple2<String, Integer>(lineSplited[0], 
                    //Integer.valueOf()可以将基本类型int转换为包装类型Integer，或者将String转换成Integer，String如果为Null或“”都会报错;
                                Integer.valueOf(lineSplited[1]));
                    }
                    
                });
        
        JavaPairRDD<String, Iterable<Integer>> groupedPairs = pairs.groupByKey();
        
        JavaPairRDD<String, Iterable<Integer>> top3Score = groupedPairs.mapToPair(
                
                new PairFunction<Tuple2<String,Iterable<Integer>>, String, Iterable<Integer>>() {

                    private static final long serialVersionUID = 1L;

                    @Override
                    public Tuple2<String, Iterable<Integer>> call(
                            Tuple2<String, Iterable<Integer>> classScores)
                            throws Exception {
                        Integer[] top3 = new Integer[3];
                        
                        String className = classScores._1;
                        Iterator<Integer> scores = classScores._2.iterator();
                        
                        while(scores.hasNext()) {
                            Integer score = scores.next();
                            
                            for(int i = 0; i < 3; i++) {
                                if(top3[i] == null) {
                                    top3[i] = score;
                                    break;
                                } else if(score > top3[i]) {
                                    for(int j = 2; j > i; j--) {
                                        top3[j] = top3[j - 1];  
                                    }
                                    
                                    top3[i] = score;
                                    
                                    break;
                                } 
                            }
                        }
                        
                        return new Tuple2<String, 
                                Iterable<Integer>>(className, Arrays.asList(top3));    
                    }
                    
                });
        
        top3Score.foreach(new VoidFunction<Tuple2<String,Iterable<Integer>>>() {
            
            private static final long serialVersionUID = 1L;

            @Override
            public void call(Tuple2<String, Iterable<Integer>> t) throws Exception {
                System.out.println("class: " + t._1);  
                Iterator<Integer> scoreIterator = t._2.iterator();
                while(scoreIterator.hasNext()) {
                    Integer score = scoreIterator.next();
                    System.out.println(score);  
                }
                System.out.println("=======================================");   
            }
            
        });
        
        sc.close();
    }
    
}





-----scala实现------

package cn.spark.study.core

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext

object GroupTop3 {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("GroupTop3Scala").setMaster("local")
    val context = new SparkContext(conf)
    val linesRDD = context.textFile("D:\test-file\score.txt")
    val studentScores = linesRDD.map(line => (line.split(" ")(0), line.split(" ")(1).toInt))
    val groupStudentScores = studentScores.groupByKey()
    val result = groupStudentScores.map(student => {
      val maxScore = new Array[Int](3)
      val scores = student._2
      for(score <- scores) {
        var flag = true
        for(i <- 0 until maxScore.length if flag) {
          if(maxScore(i) == Nil) {
            maxScore(i) = score
            flag = false
          }else{
            if(maxScore(i) < score) {
              for(j <-  (i + 1 to maxScore.length - 1).reverse){
                maxScore(j) = maxScore(j - 1)
              }
              maxScore(i) = score
              flag = false
            }
          }
        }
      }
      (student._1, maxScore)
    })
 
    result.foreach(result =>{
      print(result._1 + "班级前三明成绩为：")
      for(i <- 0 until result._2.length) {
        if(i == 0) print(result._2(i))
        else print("," + result._2(i))
      }
      println()
    })
  }
}