先看
AbstractFetcher
这个可以理解就是，consumer中具体去kafka读数据的线程，一个fetcher可以同时读多个partitions的数据来看看
/** * Base class for all fetchers, which implement the connections to Kafka brokers and * pull records from Kafka partitions. * * <p>This fetcher base class implements the logic around emitting records and tracking offsets, * as well as around the optional timestamp assignment and watermark generation. * * @param <T> The type of elements deserialized from Kafka's byte records, and emitted into * the Flink data streams. * @param <KPH> The type of topic/partition identifier used by Kafka in the specific version. */ public abstract class AbstractFetcher<T, KPH> { /** The source context to emit records and watermarks to */ private final SourceContext<T> sourceContext; //用于发送数据的context /** All partitions (and their state) that this fetcher is subscribed to */ private final KafkaTopicPartitionState<KPH>[] allPartitions; //用于记录每个topic partition的状态，比如offset // ------------------------------------------------------------------------ protected AbstractFetcher( SourceContext<T> sourceContext, List<KafkaTopicPartition> assignedPartitions, SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic, SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated, StreamingRuntimeContext runtimeContext) throws Exception { // create our partition state according to the timestamp/watermark mode this.allPartitions = initializePartitions( assignedPartitions, timestampWatermarkMode, watermarksPeriodic, watermarksPunctuated, runtimeContext.getUserCodeClassLoader()); // if we have periodic watermarks, kick off the interval scheduler if (timestampWatermarkMode == PERIODIC_WATERMARKS) { KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>[] parts = (KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>[]) allPartitions; PeriodicWatermarkEmitter periodicEmitter = new PeriodicWatermarkEmitter(parts, sourceContext, runtimeContext); periodicEmitter.start(); //定期的发出waterMark } } // ------------------------------------------------------------------------ // Core fetcher work methods // ------------------------------------------------------------------------ public abstract void runFetchLoop() throws Exception; //核心的函数，需要重载 // ------------------------------------------------------------------------ // Kafka version specifics // ------------------------------------------------------------------------ /** * Creates the Kafka version specific representation of the given * topic partition. * * @param partition The Flink representation of the Kafka topic partition. * @return The specific Kafka representation of the Kafka topic partition. */ public abstract KPH createKafkaPartitionHandle(KafkaTopicPartition partition);//生成KafkaPartitionHandle，这个其实是kafka中对partition的描述 /** * Commits the given partition offsets to the Kafka brokers (or to ZooKeeper for * older Kafka versions). * * @param offsets The offsets to commit to Kafka. * @throws Exception This method forwards exceptions. */ public abstract void commitSpecificOffsetsToKafka(Map<KafkaTopicPartition, Long> offsets) throws Exception; //如果commit相应的kafka offset，比如写到zk // ------------------------------------------------------------------------ // snapshot and restore the state // ------------------------------------------------------------------------ /** * Takes a snapshot of the partition offsets. * * <p>Important: This method mus be called under the checkpoint lock. * * @return A map from partition to current offset. */ public HashMap<KafkaTopicPartition, Long> snapshotCurrentState() { //产生所有partitions的snapshot，主要是offset // this method assumes that the checkpoint lock is held assert Thread.holdsLock(checkpointLock); HashMap<KafkaTopicPartition, Long> state = new HashMap<>(allPartitions.length); for (KafkaTopicPartitionState<?> partition : subscribedPartitions()) { if (partition.isOffsetDefined()) { state.put(partition.getKafkaTopicPartition(), partition.getOffset()); } } return state; } /** * Restores the partition offsets. * * @param snapshotState The offsets for the partitions */ public void restoreOffsets(HashMap<KafkaTopicPartition, Long> snapshotState) { //从checkpoint中去恢复offset for (KafkaTopicPartitionState<?> partition : allPartitions) { Long offset = snapshotState.get(partition.getKafkaTopicPartition()); if (offset != null) { partition.setOffset(offset); } } } // ------------------------------------------------------------------------ // emitting records // ------------------------------------------------------------------------ /** * * <p>Implementation Note: This method is kept brief to be JIT inlining friendly. * That makes the fast path efficient, the extended paths are called as separate methods. * * @param record The record to emit * @param partitionState The state of the Kafka partition from which the record was fetched * @param offset The offset from which the record was fetched */ protected final void emitRecord(T record, KafkaTopicPartitionState<KPH> partitionState, long offset) { //真正的emit record if (timestampWatermarkMode == NO_TIMESTAMPS_WATERMARKS) { // fast path logic, in case there are no watermarks // emit the record, using the checkpoint lock to guarantee // atomicity of record emission and offset state update synchronized (checkpointLock) { sourceContext.collect(record); //发出record partitionState.setOffset(offset); //更新local offset } } else if (timestampWatermarkMode == PERIODIC_WATERMARKS) { //如果有需要定期的watermark emitRecordWithTimestampAndPeriodicWatermark(record, partitionState, offset); } else { emitRecordWithTimestampAndPunctuatedWatermark(record, partitionState, offset); } } /** * Record emission, if a timestamp will be attached from an assigner that is * also a periodic watermark generator. */ private void emitRecordWithTimestampAndPeriodicWatermark( T record, KafkaTopicPartitionState<KPH> partitionState, long offset) { // extract timestamp - this accesses/modifies the per-partition state inside the // watermark generator instance, so we need to lock the access on the // partition state. concurrent access can happen from the periodic emitter final long timestamp; //noinspection SynchronizationOnLocalVariableOrMethodParameter synchronized (withWatermarksState) { timestamp = withWatermarksState.getTimestampForRecord(record); //调用waterMark.extractTimestamp来获取该record的event time } // emit the record with timestamp, using the usual checkpoint lock to guarantee // atomicity of record emission and offset state update synchronized (checkpointLock) { sourceContext.collectWithTimestamp(record, timestamp); //这个emit接口，会在发送record的情况下，还加上event time partitionState.setOffset(offset); } } // ------------------------------------------------------------------------ /** * The periodic watermark emitter. In its given interval, it checks all partitions for * the current event time watermark, and possibly emits the next watermark. */ private static class PeriodicWatermarkEmitter implements Triggerable { private final KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>[] allPartitions; private final SourceContext<?> emitter; private final StreamingRuntimeContext triggerContext; private final long interval; private long lastWatermarkTimestamp; //------------------------------------------------- PeriodicWatermarkEmitter( KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>[] allPartitions, SourceContext<?> emitter, StreamingRuntimeContext runtimeContext) { this.allPartitions = checkNotNull(allPartitions); this.emitter = checkNotNull(emitter); this.triggerContext = checkNotNull(runtimeContext); this.interval = runtimeContext.getExecutionConfig().getAutoWatermarkInterval(); this.lastWatermarkTimestamp = Long.MIN_VALUE; } //------------------------------------------------- public void start() { triggerContext.registerTimer(System.currentTimeMillis() + interval, this); //注册timer } @Override public void trigger(long timestamp) throws Exception { // sanity check assert Thread.holdsLock(emitter.getCheckpointLock()); long minAcrossAll = Long.MAX_VALUE; for (KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?> state : allPartitions) { // we access the current watermark for the periodic assigners under the state // lock, to prevent concurrent modification to any internal variables final long curr; //noinspection SynchronizationOnLocalVariableOrMethodParameter synchronized (state) { curr = state.getCurrentWatermarkTimestamp(); //获取waterMark } minAcrossAll = Math.min(minAcrossAll, curr); } // emit next watermark, if there is one if (minAcrossAll > lastWatermarkTimestamp) { lastWatermarkTimestamp = minAcrossAll; emitter.emitWatermark(new Watermark(minAcrossAll)); //emit waterMark } // schedule the next watermark triggerContext.registerTimer(System.currentTimeMillis() + interval, this); //再次注册timer } } }
Kafka08Fetcher
基于kafka 0.8版本的fetcher，
public class Kafka08Fetcher<T> extends AbstractFetcher<T, TopicAndPartition>
核心的函数，是重写
runFetchLoop
@Override public void runFetchLoop() throws Exception { // the map from broker to the thread that is connected to that broker final Map<Node, SimpleConsumerThread<T>> brokerToThread = new HashMap<>(); //cache每个partition node到每个SimpleConsumerThread的对应关系 // the offset handler handles the communication with ZooKeeper, to commit externally visible offsets final ZookeeperOffsetHandler zookeeperOffsetHandler = new ZookeeperOffsetHandler(kafkaConfig); //Zookeeper Handler，用于r/w数据到zookeeper this.zookeeperOffsetHandler = zookeeperOffsetHandler; PeriodicOffsetCommitter periodicCommitter = null; try { // read offsets from ZooKeeper for partitions that did not restore offsets { List<KafkaTopicPartition> partitionsWithNoOffset = new ArrayList<>(); for (KafkaTopicPartitionState<TopicAndPartition> partition : subscribedPartitions()) { if (!partition.isOffsetDefined()) { //遍历每个partition，如果没有定义offset，即offset没有从checkpoint中恢复，加入partitionsWithNoOffset partitionsWithNoOffset.add(partition.getKafkaTopicPartition()); } }
//这步仅仅对于没有从ckeckpoint中读到offset的partitionsWithNoOffset Map<KafkaTopicPartition, Long> zkOffsets = zookeeperOffsetHandler.getOffsets(partitionsWithNoOffset); //从zk中读出，相应partition的offset for (KafkaTopicPartitionState<TopicAndPartition> partition : subscribedPartitions()) { Long offset = zkOffsets.get(partition.getKafkaTopicPartition()); if (offset != null) { partition.setOffset(offset); //为partition设置从zk中读出的offset } } } // start the periodic offset committer thread, if necessary if (autoCommitInterval > 0) { //定期触发commit offsets，比如发送到zk，路径，topic_groupid + "/" + partition; periodicCommitter = new PeriodicOffsetCommitter(zookeeperOffsetHandler, subscribedPartitions(), errorHandler, autoCommitInterval); periodicCommitter.setName("Periodic Kafka partition offset committer"); periodicCommitter.setDaemon(true); periodicCommitter.start(); } // Main loop polling elements from the unassignedPartitions queue to the threads while (running) { // wait for max 5 seconds trying to get partitions to assign // if threads shut down, this poll returns earlier, because the threads inject the // special marker into the queue List<KafkaTopicPartitionState<TopicAndPartition>> partitionsToAssign = unassignedPartitionsQueue.getBatchBlocking(5000); partitionsToAssign.remove(MARKER); //这边这个marker干嘛用的。。。，防止上面被block？ if (!partitionsToAssign.isEmpty()) { LOG.info("Assigning {} partitions to broker threads", partitionsToAssign.size()); Map<Node, List<KafkaTopicPartitionState<TopicAndPartition>>> partitionsWithLeaders = //通过broker server找到partitions的leader，返回的结果，map（leader <-> partition list） findLeaderForPartitions(partitionsToAssign, kafkaConfig); // assign the partitions to the leaders (maybe start the threads) for (Map.Entry<Node, List<KafkaTopicPartitionState<TopicAndPartition>>> partitionsWithLeader : partitionsWithLeaders.entrySet()) { final Node leader = partitionsWithLeader.getKey(); //leader node final List<KafkaTopicPartitionState<TopicAndPartition>> partitions = partitionsWithLeader.getValue(); //这个leader node可以读取的partition列表 SimpleConsumerThread<T> brokerThread = brokerToThread.get(leader); //找到leader node对应的consumer thread if (brokerThread == null || !brokerThread.getNewPartitionsQueue().isOpen()) { // start new thread brokerThread = createAndStartSimpleConsumerThread(partitions, leader, errorHandler); //如果没有相应的consumer thread，创建新的consumer thread brokerToThread.put(leader, brokerThread); // } else { // put elements into queue of thread ClosableBlockingQueue<KafkaTopicPartitionState<TopicAndPartition>> newPartitionsQueue = // brokerThread.getNewPartitionsQueue(); for (KafkaTopicPartitionState<TopicAndPartition> fp : partitions) { if (!newPartitionsQueue.addIfOpen(fp)) { // // we were unable to add the partition to the broker's queue // the broker has closed in the meantime (the thread will shut down) // create a new thread for connecting to this broker List<KafkaTopicPartitionState<TopicAndPartition>> seedPartitions = new ArrayList<>(); seedPartitions.add(fp); brokerThread = createAndStartSimpleConsumerThread(seedPartitions, leader, errorHandler); brokerToThread.put(leader, brokerThread); newPartitionsQueue = brokerThread.getNewPartitionsQueue(); // update queue for the subsequent partitions } } } } } } } catch (InterruptedException e) { //...... } finally { //...... } }
其他一些接口实现，
// ------------------------------------------------------------------------ // Kafka 0.8 specific class instantiation // ------------------------------------------------------------------------ @Override public TopicAndPartition createKafkaPartitionHandle(KafkaTopicPartition partition) { return new TopicAndPartition(partition.getTopic(), partition.getPartition()); //对于kafka0.8，KafkaPartitionHandle就是TopicAndPartition } // ------------------------------------------------------------------------ // Offset handling // ------------------------------------------------------------------------ @Override public void commitSpecificOffsetsToKafka(Map<KafkaTopicPartition, Long> offsets) throws Exception { ZookeeperOffsetHandler zkHandler = this.zookeeperOffsetHandler; if (zkHandler != null) { zkHandler.writeOffsets(offsets); //commit offsets是写到zookeeper的 } } // ------------------------------------------------------------------------ // Utilities // ------------------------------------------------------------------------ private SimpleConsumerThread<T> createAndStartSimpleConsumerThread( List<KafkaTopicPartitionState<TopicAndPartition>> seedPartitions, Node leader, ExceptionProxy errorHandler) throws IOException, ClassNotFoundException { // each thread needs its own copy of the deserializer, because the deserializer is // not necessarily thread safe final KeyedDeserializationSchema<T> clonedDeserializer = InstantiationUtil.clone(deserializer, userCodeClassLoader); // seed thread with list of fetch partitions (otherwise it would shut down immediately again SimpleConsumerThread<T> brokerThread = new SimpleConsumerThread<>( this, errorHandler, kafkaConfig, leader, seedPartitions, unassignedPartitionsQueue, clonedDeserializer, invalidOffsetBehavior); brokerThread.setName(String.format("SimpleConsumer - %s - broker-%s (%s:%d)", taskName, leader.id(), leader.host(), leader.port())); brokerThread.setDaemon(true); brokerThread.start(); //创建和启动SimpleConsumerThread LOG.info("Starting thread {}", brokerThread.getName()); return brokerThread; }
下面来看看SimpleConsumerThread
class SimpleConsumerThread<T> extends Thread
核心函数run，主要做的是，不停的读取数据的事情，
// these are the actual configuration values of Kafka + their original default values. this.soTimeout = getInt(config, "socket.timeout.ms", 30000); //Kafka的一些配置 this.minBytes = getInt(config, "fetch.min.bytes", 1); this.maxWait = getInt(config, "fetch.wait.max.ms", 100); this.fetchSize = getInt(config, "fetch.message.max.bytes", 1048576); this.bufferSize = getInt(config, "socket.receive.buffer.bytes", 65536); this.reconnectLimit = getInt(config, "flink.simple-consumer-reconnectLimit", 3); // ------------------------------------------------------------------------ // main work loop // ------------------------------------------------------------------------ @Override public void run() { try { // create the Kafka consumer that we actually use for fetching consumer = new SimpleConsumer(broker.host(), broker.port(), soTimeout, bufferSize, clientId); //创建SimpleConsumer // make sure that all partitions have some offsets to start with // those partitions that do not have an offset from a checkpoint need to get // their start offset from ZooKeeper getMissingOffsetsFromKafka(partitions); //为没有offset信息的partition，重置offset，从latest或earlist // Now, the actual work starts :-) int offsetOutOfRangeCount = 0; //用于统计实际执行情况，非法offset，或重连的计数 int reconnects = 0; while (running) { // ----------------------------------- partitions list maintenance ---------------------------- // check queue for new partitions to read from: List<KafkaTopicPartitionState<TopicAndPartition>> newPartitions = newPartitionsQueue.pollBatch(); //对于new partitions的处理，主要就是把它们加到partitions当中 if (newPartitions != null) { // found some new partitions for this thread's broker // check if the new partitions need an offset lookup getMissingOffsetsFromKafka(newPartitions); // 为新的partition重置offset // add the new partitions (and check they are not already in there) for (KafkaTopicPartitionState<TopicAndPartition> newPartition: newPartitions) { partitions.add(newPartition); } } if (partitions.size() == 0) { //如果partitions为空，即没有需要消费的partition if (newPartitionsQueue.close()) { //如果此时newPartitionsQueue为closed，那么就不可能会有新的partitions加入，那么该thread就没有存在的意义，不需要继续run // close succeeded. Closing thread running = false; //关闭线程 LOG.info("Consumer thread {} does not have any partitions assigned anymore. Stopping thread.", getName()); // add the wake-up marker into the queue to make the main thread // immediately wake up and termination faster unassignedPartitions.add(MARKER); break; } else { //如果newPartitionsQueue没有被关闭，那就等待新的partitions，continue // close failed: fetcher main thread concurrently added new partitions into the queue. // go to top of loop again and get the new partitions continue; } } // ----------------------------------- request / response with kafka ---------------------------- FetchRequestBuilder frb = new FetchRequestBuilder(); //创建FetchRequestBuilder frb.clientId(clientId); frb.maxWait(maxWait); frb.minBytes(minBytes); for (KafkaTopicPartitionState<?> partition : partitions) { frb.addFetch( partition.getKafkaTopicPartition().getTopic(), partition.getKafkaTopicPartition().getPartition(), partition.getOffset() + 1, // request the next record fetchSize); } kafka.api.FetchRequest fetchRequest = frb.build(); //创建FetchRequest，一个request可以同时读多个partition，取决于partition和consumer数量的比例 FetchResponse fetchResponse; try { fetchResponse = consumer.fetch(fetchRequest); //从kafka读到数据，包含在FetchResponse中，其中包含从多个partition中取到的数据 } catch (Throwable cce) { //noinspection ConstantConditions if (cce instanceof ClosedChannelException) { //链接kafka异常 // we don't know if the broker is overloaded or unavailable. // retry a few times, then return ALL partitions for new leader lookup if (++reconnects >= reconnectLimit) { //如果达到重连limit，说明确实无法连接到kafka LOG.warn("Unable to reach broker after {} retries. Returning all current partitions", reconnectLimit); for (KafkaTopicPartitionState<TopicAndPartition> fp: this.partitions) { unassignedPartitions.add(fp); //把负责的partitions加入unassignedPartitions，表明这些partition是没人处理的 } this.partitions.clear(); //把partitons清空 continue; // jump to top of loop: will close thread or subscribe to new partitions //这步会走到上面的partitions.size() == 0的逻辑 } try { //如果需要重试，先关闭consumer，然后重新创建consumer，然后continue重试 consumer.close(); } catch (Throwable t) { LOG.warn("Error while closing consumer connection", t); } // delay & retry Thread.sleep(100); consumer = new SimpleConsumer(broker.host(), broker.port(), soTimeout, bufferSize, clientId); continue; // retry } else { throw cce; } } reconnects = 0; // ---------------------------------------- error handling ---------------------------- if (fetchResponse == null) { throw new IOException("Fetch from Kafka failed (request returned null)"); } if (fetchResponse.hasError()) { //如果fetchResponse有错误 String exception = ""; List<KafkaTopicPartitionState<TopicAndPartition>> partitionsToGetOffsetsFor = new ArrayList<>(); // iterate over partitions to get individual error codes Iterator<KafkaTopicPartitionState<TopicAndPartition>> partitionsIterator = partitions.iterator(); boolean partitionsRemoved = false; while (partitionsIterator.hasNext()) { final KafkaTopicPartitionState<TopicAndPartition> fp = partitionsIterator.next(); short code = fetchResponse.errorCode(fp.getTopic(), fp.getPartition()); //取得对于该partition的error code if (code == ErrorMapping.OffsetOutOfRangeCode()) { //非法offset，那么需要重新初始化该partition的offset // we were asked to read from an out-of-range-offset (maybe set wrong in Zookeeper) // Kafka's high level consumer is resetting the offset according to 'auto.offset.reset' partitionsToGetOffsetsFor.add(fp); } else if (code == ErrorMapping.NotLeaderForPartitionCode() || //如果由于各种不可用，导致无法从该broker上读取到partition的数据 code == ErrorMapping.LeaderNotAvailableCode() || code == ErrorMapping.BrokerNotAvailableCode() || code == ErrorMapping.UnknownCode()) { // the broker we are connected to is not the leader for the partition. LOG.warn("{} is not the leader of {}. Reassigning leader for partition", broker, fp); LOG.debug("Error code = {}", code); unassignedPartitions.add(fp); //那么把该partition放回unassignedPartitions，等待重新分配 partitionsIterator.remove(); // unsubscribe the partition ourselves，从当前的partitions列表中把该partition删除 partitionsRemoved = true; } else if (code != ErrorMapping.NoError()) { exception += " Exception for " + fp.getTopic() +":"+ fp.getPartition() + ": " + StringUtils.stringifyException(ErrorMapping.exceptionFor(code)); } } if (partitionsToGetOffsetsFor.size() > 0) { // safeguard against an infinite loop. if (offsetOutOfRangeCount++ > 3) { //如果对于partitions，3次重置offset后，offset仍然有非法的，抛异常，防止无限循环 throw new RuntimeException("Found invalid offsets more than three times in partitions " + partitionsToGetOffsetsFor + " Exceptions: " + exception); } // get valid offsets for these partitions and try again. LOG.warn("The following partitions had an invalid offset: {}", partitionsToGetOffsetsFor); getLastOffsetFromKafka(consumer, partitionsToGetOffsetsFor, invalidOffsetBehavior); //重置这些partitions的offset, 根据配置会reset到earliest或latest LOG.warn("The new partition offsets are {}", partitionsToGetOffsetsFor); continue; // jump back to create a new fetch request. The offset has not been touched. } else if (partitionsRemoved) { continue; // create new fetch request } else { // partitions failed on an error throw new IOException("Error while fetching from broker '" + broker +"': " + exception); } } else { // successful fetch, reset offsetOutOfRangeCount. offsetOutOfRangeCount = 0; } // ----------------------------------- process fetch response ---------------------------- int messagesInFetch = 0; int deletedMessages = 0; Iterator<KafkaTopicPartitionState<TopicAndPartition>> partitionsIterator = partitions.iterator(); partitionsLoop: while (partitionsIterator.hasNext()) { final KafkaTopicPartitionState<TopicAndPartition> currentPartition = partitionsIterator.next(); final ByteBufferMessageSet messageSet = fetchResponse.messageSet( //取出fetchResponse关于该partition的数据，封装成ByteBufferMessageSet currentPartition.getTopic(), currentPartition.getPartition()); for (MessageAndOffset msg : messageSet) { //对于每天message if (running) { messagesInFetch++; final ByteBuffer payload = msg.message().payload(); //读出message内容 final long offset = msg.offset(); //读出message offset if (offset <= currentPartition.getOffset()) { //旧数据，ignore // we have seen this message already LOG.info("Skipping message with offset " + msg.offset() + " because we have seen messages until (including) " + currentPartition.getOffset() + " from topic/partition " + currentPartition.getTopic() + '/' + currentPartition.getPartition() + " already"); continue; } // If the message value is null, this represents a delete command for the message key. // Log this and pass it on to the client who might want to also receive delete messages. byte[] valueBytes; if (payload == null) { deletedMessages++; valueBytes = null; } else { valueBytes = new byte[payload.remaining()]; payload.get(valueBytes); //将内容，读入valueBytes } // put key into byte array byte[] keyBytes = null; int keySize = msg.message().keySize(); if (keySize >= 0) { // message().hasKey() is doing the same. We save one int deserialization ByteBuffer keyPayload = msg.message().key(); keyBytes = new byte[keySize]; //将key读入keyBytes keyPayload.get(keyBytes); } final T value = deserializer.deserialize(keyBytes, valueBytes, //将message反序列化成对象 currentPartition.getTopic(), currentPartition.getPartition(), offset); if (deserializer.isEndOfStream(value)) { // remove partition from subscribed partitions. partitionsIterator.remove(); continue partitionsLoop; } owner.emitRecord(value, currentPartition, offset); //emit 数据 } else { // no longer running return; } } } } // end of fetch loop } }
最后，看看
FlinkKafkaConsumerBase
/** * Base class of all Flink Kafka Consumer data sources. * This implements the common behavior across all Kafka versions. * * <p>The Kafka version specific behavior is defined mainly in the specific subclasses of the * {@link AbstractFetcher}. * * @param <T> The type of records produced by this data source */ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFunction<T> implements CheckpointListener, CheckpointedAsynchronously<HashMap<KafkaTopicPartition, Long>>, ResultTypeQueryable<T>
这个是对所有版本kafka的抽象，
@Override public void run(SourceContext<T> sourceContext) throws Exception { // figure out which partitions this subtask should process final List<KafkaTopicPartition> thisSubtaskPartitions = assignPartitions(allSubscribedPartitions, //对应于topic partition数和consumer数，一个consumer应该分配哪些partitions，这里逻辑就是简单的取模 getRuntimeContext().getNumberOfParallelSubtasks(), getRuntimeContext().getIndexOfThisSubtask()); // we need only do work, if we actually have partitions assigned if (!thisSubtaskPartitions.isEmpty()) { // (1) create the fetcher that will communicate with the Kafka brokers final AbstractFetcher<T, ?> fetcher = createFetcher( //创建Fetcher sourceContext, thisSubtaskPartitions, periodicWatermarkAssigner, punctuatedWatermarkAssigner, (StreamingRuntimeContext) getRuntimeContext()); // (2) set the fetcher to the restored checkpoint offsets if (restoreToOffset != null) { //这个如果从checkpoint中读出offset状态 fetcher.restoreOffsets(restoreToOffset); //恢复offset } // publish the reference, for snapshot-, commit-, and cancel calls // IMPORTANT: We can only do that now, because only now will calls to // the fetchers 'snapshotCurrentState()' method return at least // the restored offsets this.kafkaFetcher = fetcher; if (!running) { return; } // (3) run the fetcher' main work method fetcher.runFetchLoop(); //开始run fetcher } } @Override public HashMap<KafkaTopicPartition, Long> snapshotState(long checkpointId, long checkpointTimestamp) throws Exception { final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher; HashMap<KafkaTopicPartition, Long> currentOffsets = fetcher.snapshotCurrentState(); //snapshot当前的offset // the map cannot be asynchronously updated, because only one checkpoint call can happen // on this function at a time: either snapshotState() or notifyCheckpointComplete() pendingCheckpoints.put(checkpointId, currentOffsets); //cache当前的checkpointid，等待该checkpoint完成 // truncate the map, to prevent infinite growth while (pendingCheckpoints.size() > MAX_NUM_PENDING_CHECKPOINTS) { //删除过期的，或老的checkpoints pendingCheckpoints.remove(0); } return currentOffsets; } @Override public void restoreState(HashMap<KafkaTopicPartition, Long> restoredOffsets) { LOG.info("Setting restore state in the FlinkKafkaConsumer"); restoreToOffset = restoredOffsets; } @Override public void notifyCheckpointComplete(long checkpointId) throws Exception { //当这个checkpoint完成时，需要通知kafkaconsumer，这个时候才会真正的commit offset final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher; try { final int posInMap = pendingCheckpoints.indexOf(checkpointId); @SuppressWarnings("unchecked") HashMap<KafkaTopicPartition, Long> checkpointOffsets = (HashMap<KafkaTopicPartition, Long>) pendingCheckpoints.remove(posInMap); // remove older checkpoints in map for (int i = 0; i < posInMap; i++) { //比该checkpoint老的未完成的checkpoint已经没有意义，删除 pendingCheckpoints.remove(0); } fetcher.commitSpecificOffsetsToKafka(checkpointOffsets); //真正的commit offset，这个是通用接口，虽然对于kafka0.8，Fetcher里面本身也是会定期提交的，checkpoint一般秒级别比定期提交更频繁些 } } /** * Selects which of the given partitions should be handled by a specific consumer, * given a certain number of consumers. * * @param allPartitions The partitions to select from * @param numConsumers The number of consumers * @param consumerIndex The index of the specific consumer * * @return The sublist of partitions to be handled by that consumer. */ protected static List<KafkaTopicPartition> assignPartitions( List<KafkaTopicPartition> allPartitions, int numConsumers, int consumerIndex) { final List<KafkaTopicPartition> thisSubtaskPartitions = new ArrayList<>( allPartitions.size() / numConsumers + 1); for (int i = 0; i < allPartitions.size(); i++) { if (i % numConsumers == consumerIndex) { thisSubtaskPartitions.add(allPartitions.get(i)); } } return thisSubtaskPartitions; }
针对kafka0.8的consumer
public class FlinkKafkaConsumer08<T> extends FlinkKafkaConsumerBase<T> { /** * Creates a new Kafka streaming source consumer for Kafka 0.8.x * * This constructor allows passing multiple topics and a key/value deserialization schema. * * @param topics * The Kafka topics to read from. * @param deserializer * The keyed de-/serializer used to convert between Kafka's byte messages and Flink's objects. * @param props * The properties that are used to configure both the fetcher and the offset handler. */ public FlinkKafkaConsumer08(List<String> topics, KeyedDeserializationSchema<T> deserializer, Properties props) { super(deserializer); // validate the zookeeper properties validateZooKeeperConfig(props); this.invalidOffsetBehavior = getInvalidOffsetBehavior(props); //当offset非法的时候，选择从哪里重置，这里支持earlist或latest this.autoCommitInterval = PropertiesUtil.getLong(props, "auto.commit.interval.ms", 60000); //offset commit的间隔，默认是1分钟 // Connect to a broker to get the partitions for all topics List<KafkaTopicPartition> partitionInfos = KafkaTopicPartition.dropLeaderData(getPartitionsForTopic(topics, props)); //这里只是取出topic相关的partition的信息 setSubscribedPartitions(partitionInfos); //将这部分，即该consumer消费的partitions，加入到SubscribedPartitions，表明这些已经有consumer消费了 } /** * Send request to Kafka to get partitions for topic. * * @param topics The name of the topics. * @param properties The properties for the Kafka Consumer that is used to query the partitions for the topic. */ public static List<KafkaTopicPartitionLeader> getPartitionsForTopic(List<String> topics, Properties properties) { //这里的逻辑是如果从kafka取得topic的partititon信息，这里配置配的是broker list，而非zk，所以他每次会随机从所有的brokers中挑一个去读取partitions信息 String seedBrokersConfString = properties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG); final int numRetries = getInt(properties, GET_PARTITIONS_RETRIES_KEY, DEFAULT_GET_PARTITIONS_RETRIES); checkNotNull(seedBrokersConfString, "Configuration property %s not set", ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG); String[] seedBrokers = seedBrokersConfString.split(","); List<KafkaTopicPartitionLeader> partitions = new ArrayList<>(); final String clientId = "flink-kafka-consumer-partition-lookup"; final int soTimeout = getInt(properties, "socket.timeout.ms", 30000); final int bufferSize = getInt(properties, "socket.receive.buffer.bytes", 65536); Random rnd = new Random(); retryLoop: for (int retry = 0; retry < numRetries; retry++) { // we pick a seed broker randomly to avoid overloading the first broker with all the requests when the // parallel source instances start. Still, we try all available brokers. int index = rnd.nextInt(seedBrokers.length); brokersLoop: for (int arrIdx = 0; arrIdx < seedBrokers.length; arrIdx++) { String seedBroker = seedBrokers[index]; LOG.info("Trying to get topic metadata from broker {} in try {}/{}", seedBroker, retry, numRetries); if (++index == seedBrokers.length) { index = 0; } URL brokerUrl = NetUtils.getCorrectHostnamePort(seedBroker); SimpleConsumer consumer = null; try { consumer = new SimpleConsumer(brokerUrl.getHost(), brokerUrl.getPort(), soTimeout, bufferSize, clientId); TopicMetadataRequest req = new TopicMetadataRequest(topics); kafka.javaapi.TopicMetadataResponse resp = consumer.send(req); List<TopicMetadata> metaData = resp.topicsMetadata(); // clear in case we have an incomplete list from previous tries partitions.clear(); for (TopicMetadata item : metaData) { if (item.errorCode() != ErrorMapping.NoError()) { // warn and try more brokers LOG.warn("Error while getting metadata from broker " + seedBroker + " to find partitions " + "for " + topics.toString() + ". Error: " + ErrorMapping.exceptionFor(item.errorCode()).getMessage()); continue brokersLoop; } if (!topics.contains(item.topic())) { LOG.warn("Received metadata from topic " + item.topic() + " even though it was not requested. Skipping ..."); continue brokersLoop; } for (PartitionMetadata part : item.partitionsMetadata()) { Node leader = brokerToNode(part.leader()); KafkaTopicPartition ktp = new KafkaTopicPartition(item.topic(), part.partitionId()); KafkaTopicPartitionLeader pInfo = new KafkaTopicPartitionLeader(ktp, leader); partitions.add(pInfo); } } break retryLoop; // leave the loop through the brokers } catch (Exception e) { LOG.warn("Error communicating with broker " + seedBroker + " to find partitions for " + topics.toString() + "." + "" + e.getClass() + ". Message: " + e.getMessage()); LOG.debug("Detailed trace", e); // we sleep a bit. Retrying immediately doesn't make sense in cases where Kafka is reorganizing the leader metadata try { Thread.sleep(500); } catch (InterruptedException e1) { // sleep shorter. } } finally { if (consumer != null) { consumer.close(); } } } // brokers loop } // retries loop return partitions; } private static long getInvalidOffsetBehavior(Properties config) { final String val = config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "largest"); if (val.equals("none")) { throw new IllegalArgumentException("Cannot use '" + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG + "' value 'none'. Possible values: 'latest', 'largest', or 'earliest'."); } else if (val.equals("largest") || val.equals("latest")) { // largest is kafka 0.8, latest is kafka 0.9 return OffsetRequest.LatestTime(); } else { return OffsetRequest.EarliestTime(); } }