• 数据零丢失kafka + checkpoint


    package kafkautils

    /**
    * Created on 上午12:48.
    *
    * High level comsumer api
    *
    * low level comsumer api(simple comsumer api)
    *
    *
    */
    import kafka.serializer.StringDecoder
    import org.apache.spark.SparkConf
    import org.apache.spark.streaming.kafka.KafkaUtils
    import org.apache.spark.streaming.{Duration, Seconds, StreamingContext}


    object StreamingWithCheckpoint {
    def main(args: Array[String]) {
    //val Array(brokers, topics) = args
    val processingInterval = 2
    val brokers = "spark123:9092"
    val topics = "mytest1"
    // Create context with 2 second batch interval
    val sparkConf = new SparkConf().setAppName("ConsumerWithCheckPoint").setMaster("local[2]")
    // Create direct kafka stream with brokers and topics
    val topicsSet = topics.split(",").toSet
    val kafkaParams = Map[String, String]("metadata.broker.list" -> brokers,
    "auto.offset.reset" -> "smallest")
    val checkpointPath = "hdfs://spark123:8020/spark_checkpoint10"
    def functionToCreateContext(): StreamingContext = {
    val ssc = new StreamingContext(sparkConf, Seconds(processingInterval))
    val messages = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParams, topicsSet)

    ssc.checkpoint(checkpointPath)
    messages.checkpoint(Duration(8*processingInterval.toInt*1000))
    messages.foreachRDD(rdd => {
    if(!rdd.isEmpty()){
    println("################################" + rdd.count())
    }

    })
    ssc
    }

    // 如果有checkpoint则checkpoint中记录的信息恢复StreamingContext
    val context = StreamingContext.getOrCreate(checkpointPath, functionToCreateContext _)
    context.start()
    context.awaitTermination()
    }
    }

  • 相关阅读:
    图论基础
    排序二叉树的中序遍历
    先序遍历序列和中序遍历序列,输出该二叉树的后序遍历序列
    数据结构实验之 二叉树的建立与遍历
    c++ 头文件 及 sort 和 vector简单介绍
    最短路径(Floyd 模板题)
    最小生成树(kruskal模版 模板)
    基于邻接矩阵的广度优先搜索遍历(BFS)
    [SCOI2015]国旗计划
    [HNOI2015]开店
  • 原文地址:https://www.cnblogs.com/heguoxiu/p/10149647.html
Copyright © 2020-2023  润新知