spark accumulator累加器

java

/**
 * accumulator可以让多个task共同操作一份变量,主要进行多个节点对一个变量进行共享性的操作,accumulator只提供了累加的功能
 * 只有driver可以获取accumulator的值
 * @author Tele
 */
public class AccumulatorDemo {
    private static SparkConf conf = new SparkConf().setMaster("local").setAppName("AccumulatorDemo");
    private static JavaSparkContext jsc = new JavaSparkContext(conf);

    public static void main(String[] args) {
        List<Integer> list = Arrays.asList(1, 2, 3, 4, 5, 6);

        JavaRDD<Integer> rdd = jsc.parallelize(list);

        /*
         * Accumulator<Integer> accumulator = jsc.accumulator(10);
         * 
         * rdd.foreach(new VoidFunction<Integer>() {
         * 
         * private static final long serialVersionUID = 1L;
         * 
         * @Override public void call(Integer t) throws Exception { accumulator.add(t);
         * } }); System.out.println(accumulator.value());
         */

        LongAccumulator la = new LongAccumulator();
        la.setValue(100L);

        jsc.sc().register(la, "数值累加器");

        rdd.foreach(new VoidFunction<Integer>() {

            private static final long serialVersionUID = 1L;

            @Override
            public void call(Integer t) throws Exception {
                // 不能在算子内部获得accumulator.value()
                la.add(t);
            }
        });

        System.out.println(la.value());
        jsc.close();
    }
}

scala

object AccumulatorDemo {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("accumulator");
    val sc = new SparkContext(conf);

    val arr = Array(1, 2, 3, 4, 5);
    val rdd = sc.parallelize(arr, 1);

    val accumulator = new LongAccumulator;
    accumulator.add(100);

    sc.register(accumulator);

    rdd.foreach(accumulator.add(_));

    println(accumulator.value);

  }
}