Flink - FlinkKafkaConsumer08

 

先看

AbstractFetcher

这个可以理解就是，consumer中具体去kafka读数据的线程，一个fetcher可以同时读多个partitions的数据来看看

/**
 * Base class for all fetchers, which implement the connections to Kafka brokers and
 * pull records from Kafka partitions.
 * 
 * <p>This fetcher base class implements the logic around emitting records and tracking offsets,
 * as well as around the optional timestamp assignment and watermark generation. 
 * 
 * @param <T> The type of elements deserialized from Kafka's byte records, and emitted into
 *            the Flink data streams.
 * @param <KPH> The type of topic/partition identifier used by Kafka in the specific version.
 */
public abstract class AbstractFetcher<T, KPH> {
    
    /** The source context to emit records and watermarks to */
    private final SourceContext<T> sourceContext; //用于发送数据的context

    /** All partitions (and their state) that this fetcher is subscribed to */
    private final KafkaTopicPartitionState<KPH>[] allPartitions; //用于记录每个topic partition的状态，比如offset


    // ------------------------------------------------------------------------
    
    protected AbstractFetcher(
            SourceContext<T> sourceContext,
            List<KafkaTopicPartition> assignedPartitions,
            SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
            SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
            StreamingRuntimeContext runtimeContext) throws Exception
    {
        // create our partition state according to the timestamp/watermark mode 
        this.allPartitions = initializePartitions(
                assignedPartitions,
                timestampWatermarkMode,
                watermarksPeriodic, watermarksPunctuated,
                runtimeContext.getUserCodeClassLoader());
        
        // if we have periodic watermarks, kick off the interval scheduler
        if (timestampWatermarkMode == PERIODIC_WATERMARKS) {
            KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>[] parts = 
                    (KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>[]) allPartitions;
            
            PeriodicWatermarkEmitter periodicEmitter = 
                    new PeriodicWatermarkEmitter(parts, sourceContext, runtimeContext);
            periodicEmitter.start();  //定期的发出waterMark
        }
    }

    // ------------------------------------------------------------------------
    //  Core fetcher work methods
    // ------------------------------------------------------------------------

    public abstract void runFetchLoop() throws Exception; //核心的函数，需要重载
    
    // ------------------------------------------------------------------------
    //  Kafka version specifics
    // ------------------------------------------------------------------------
    
    /**
     * Creates the Kafka version specific representation of the given
     * topic partition.
     * 
     * @param partition The Flink representation of the Kafka topic partition.
     * @return The specific Kafka representation of the Kafka topic partition.
     */
    public abstract KPH createKafkaPartitionHandle(KafkaTopicPartition partition);//生成KafkaPartitionHandle，这个其实是kafka中对partition的描述

    /**
     * Commits the given partition offsets to the Kafka brokers (or to ZooKeeper for
     * older Kafka versions).
     * 
     * @param offsets The offsets to commit to Kafka.
     * @throws Exception This method forwards exceptions.
     */
    public abstract void commitSpecificOffsetsToKafka(Map<KafkaTopicPartition, Long> offsets) throws Exception; //如果commit相应的kafka offset，比如写到zk

    // ------------------------------------------------------------------------
    //  snapshot and restore the state
    // ------------------------------------------------------------------------

    /**
     * Takes a snapshot of the partition offsets.
     * 
     * <p>Important: This method mus be called under the checkpoint lock.
     * 
     * @return A map from partition to current offset.
     */
    public HashMap<KafkaTopicPartition, Long> snapshotCurrentState() { //产生所有partitions的snapshot，主要是offset
        // this method assumes that the checkpoint lock is held
        assert Thread.holdsLock(checkpointLock);

        HashMap<KafkaTopicPartition, Long> state = new HashMap<>(allPartitions.length);
        for (KafkaTopicPartitionState<?> partition : subscribedPartitions()) {
            if (partition.isOffsetDefined()) {
                state.put(partition.getKafkaTopicPartition(), partition.getOffset());
            }
        }
        return state;
    }

    /**
     * Restores the partition offsets.
     * 
     * @param snapshotState The offsets for the partitions 
     */
    public void restoreOffsets(HashMap<KafkaTopicPartition, Long> snapshotState) { //从checkpoint中去恢复offset
        for (KafkaTopicPartitionState<?> partition : allPartitions) {
            Long offset = snapshotState.get(partition.getKafkaTopicPartition());
            if (offset != null) {
                partition.setOffset(offset);
            }
        }
    }
    
    // ------------------------------------------------------------------------
    //  emitting records
    // ------------------------------------------------------------------------

    /**
     * 
     * <p>Implementation Note: This method is kept brief to be JIT inlining friendly.
     * That makes the fast path efficient, the extended paths are called as separate methods.
     * 
     * @param record The record to emit
     * @param partitionState The state of the Kafka partition from which the record was fetched
     * @param offset The offset from which the record was fetched
     */
    protected final void emitRecord(T record, KafkaTopicPartitionState<KPH> partitionState, long offset) { //真正的emit record
        if (timestampWatermarkMode == NO_TIMESTAMPS_WATERMARKS) {
            // fast path logic, in case there are no watermarks

            // emit the record, using the checkpoint lock to guarantee
            // atomicity of record emission and offset state update
            synchronized (checkpointLock) {
                sourceContext.collect(record); //发出record
                partitionState.setOffset(offset); //更新local offset
            }
        }
        else if (timestampWatermarkMode == PERIODIC_WATERMARKS) { //如果有需要定期的watermark
            emitRecordWithTimestampAndPeriodicWatermark(record, partitionState, offset);
        }
        else {
            emitRecordWithTimestampAndPunctuatedWatermark(record, partitionState, offset);
        }
    }

    /**
     * Record emission, if a timestamp will be attached from an assigner that is
     * also a periodic watermark generator.
     */
    private void emitRecordWithTimestampAndPeriodicWatermark(
            T record, KafkaTopicPartitionState<KPH> partitionState, long offset)
    {

        // extract timestamp - this accesses/modifies the per-partition state inside the
        // watermark generator instance, so we need to lock the access on the
        // partition state. concurrent access can happen from the periodic emitter
        final long timestamp;
        //noinspection SynchronizationOnLocalVariableOrMethodParameter
        synchronized (withWatermarksState) {
            timestamp = withWatermarksState.getTimestampForRecord(record); //调用waterMark.extractTimestamp来获取该record的event time
        }

        // emit the record with timestamp, using the usual checkpoint lock to guarantee
        // atomicity of record emission and offset state update 
        synchronized (checkpointLock) {
            sourceContext.collectWithTimestamp(record, timestamp); //这个emit接口，会在发送record的情况下，还加上event time
            partitionState.setOffset(offset);
        }
    }

 
    
    // ------------------------------------------------------------------------
    
    /**
     * The periodic watermark emitter. In its given interval, it checks all partitions for
     * the current event time watermark, and possibly emits the next watermark.
     */
    private static class PeriodicWatermarkEmitter implements Triggerable {

        private final KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>[] allPartitions;
        
        private final SourceContext<?> emitter;
        
        private final StreamingRuntimeContext triggerContext;

        private final long interval;
        
        private long lastWatermarkTimestamp;
        
        //-------------------------------------------------

        PeriodicWatermarkEmitter(
                KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>[] allPartitions,
                SourceContext<?> emitter,
                StreamingRuntimeContext runtimeContext)
        {
            this.allPartitions = checkNotNull(allPartitions);
            this.emitter = checkNotNull(emitter);
            this.triggerContext = checkNotNull(runtimeContext);
            this.interval = runtimeContext.getExecutionConfig().getAutoWatermarkInterval();
            this.lastWatermarkTimestamp = Long.MIN_VALUE;
        }

        //-------------------------------------------------
        
        public void start() {
            triggerContext.registerTimer(System.currentTimeMillis() + interval, this); //注册timer
        }
        
        @Override
        public void trigger(long timestamp) throws Exception {
            // sanity check
            assert Thread.holdsLock(emitter.getCheckpointLock());
            
            long minAcrossAll = Long.MAX_VALUE;
            for (KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?> state : allPartitions) {
                
                // we access the current watermark for the periodic assigners under the state
                // lock, to prevent concurrent modification to any internal variables
                final long curr;
                //noinspection SynchronizationOnLocalVariableOrMethodParameter
                synchronized (state) {
                    curr = state.getCurrentWatermarkTimestamp(); //获取waterMark
                }
                
                minAcrossAll = Math.min(minAcrossAll, curr);
            }
            
            // emit next watermark, if there is one
            if (minAcrossAll > lastWatermarkTimestamp) {
                lastWatermarkTimestamp = minAcrossAll;
                emitter.emitWatermark(new Watermark(minAcrossAll)); //emit waterMark
            }
            
            // schedule the next watermark
            triggerContext.registerTimer(System.currentTimeMillis() + interval, this); //再次注册timer
        }
    }
}

Kafka08Fetcher

基于kafka 0.8版本的fetcher，

public class Kafka08Fetcher<T> extends AbstractFetcher<T, TopicAndPartition>

核心的函数，是重写

runFetchLoop

@Override
    public void runFetchLoop() throws Exception {
        // the map from broker to the thread that is connected to that broker
        final Map<Node, SimpleConsumerThread<T>> brokerToThread = new HashMap<>(); //cache每个partition node到每个SimpleConsumerThread的对应关系

        // the offset handler handles the communication with ZooKeeper, to commit externally visible offsets
        final ZookeeperOffsetHandler zookeeperOffsetHandler = new ZookeeperOffsetHandler(kafkaConfig); //Zookeeper Handler，用于r/w数据到zookeeper
        this.zookeeperOffsetHandler = zookeeperOffsetHandler;

        PeriodicOffsetCommitter periodicCommitter = null;
        try {
            // read offsets from ZooKeeper for partitions that did not restore offsets
            {
                List<KafkaTopicPartition> partitionsWithNoOffset = new ArrayList<>();
                for (KafkaTopicPartitionState<TopicAndPartition> partition : subscribedPartitions()) {
                    if (!partition.isOffsetDefined()) {  //遍历每个partition，如果没有定义offset，即offset没有从checkpoint中恢复，加入partitionsWithNoOffset
                        partitionsWithNoOffset.add(partition.getKafkaTopicPartition());
                    }
                }
                

                //这步仅仅对于没有从ckeckpoint中读到offset的partitionsWithNoOffset
                Map<KafkaTopicPartition, Long> zkOffsets = zookeeperOffsetHandler.getOffsets(partitionsWithNoOffset); //从zk中读出，相应partition的offset
                for (KafkaTopicPartitionState<TopicAndPartition> partition : subscribedPartitions()) {
                    Long offset = zkOffsets.get(partition.getKafkaTopicPartition());
                    if (offset != null) {
                        partition.setOffset(offset); //为partition设置从zk中读出的offset
                    }
                }
            }

            // start the periodic offset committer thread, if necessary
            if (autoCommitInterval > 0) { //定期触发commit offsets，比如发送到zk，路径，topic_groupid + "/" + partition;
                periodicCommitter = new PeriodicOffsetCommitter(zookeeperOffsetHandler, 
                        subscribedPartitions(), errorHandler, autoCommitInterval);
                periodicCommitter.setName("Periodic Kafka partition offset committer");
                periodicCommitter.setDaemon(true);
                periodicCommitter.start();
            }

            // Main loop polling elements from the unassignedPartitions queue to the threads
            while (running) {
                
                // wait for max 5 seconds trying to get partitions to assign
                // if threads shut down, this poll returns earlier, because the threads inject the
                // special marker into the queue
                List<KafkaTopicPartitionState<TopicAndPartition>> partitionsToAssign = 
                        unassignedPartitionsQueue.getBatchBlocking(5000);
                partitionsToAssign.remove(MARKER); //这边这个marker干嘛用的。。。，防止上面被block？

                if (!partitionsToAssign.isEmpty()) {
                    LOG.info("Assigning {} partitions to broker threads", partitionsToAssign.size());
                    Map<Node, List<KafkaTopicPartitionState<TopicAndPartition>>> partitionsWithLeaders =  //通过broker server找到partitions的leader，返回的结果，map（leader <-> partition list） 
                            findLeaderForPartitions(partitionsToAssign, kafkaConfig);

                    // assign the partitions to the leaders (maybe start the threads)
                    for (Map.Entry<Node, List<KafkaTopicPartitionState<TopicAndPartition>>> partitionsWithLeader : 
                            partitionsWithLeaders.entrySet())
                    {
                        final Node leader = partitionsWithLeader.getKey(); //leader node
                        final List<KafkaTopicPartitionState<TopicAndPartition>> partitions = partitionsWithLeader.getValue(); //这个leader node可以读取的partition列表
                        SimpleConsumerThread<T> brokerThread = brokerToThread.get(leader); //找到leader node对应的consumer thread

                        if (brokerThread == null || !brokerThread.getNewPartitionsQueue().isOpen()) {
                            // start new thread
                            brokerThread = createAndStartSimpleConsumerThread(partitions, leader, errorHandler); //如果没有相应的consumer thread，创建新的consumer thread
                            brokerToThread.put(leader, brokerThread); //
                        }
                        else {
                            // put elements into queue of thread
                            ClosableBlockingQueue<KafkaTopicPartitionState<TopicAndPartition>> newPartitionsQueue =  //
                                    brokerThread.getNewPartitionsQueue();
                            
                            for (KafkaTopicPartitionState<TopicAndPartition> fp : partitions) {
                                if (!newPartitionsQueue.addIfOpen(fp)) {  //
                                    // we were unable to add the partition to the broker's queue
                                    // the broker has closed in the meantime (the thread will shut down)
                                    // create a new thread for connecting to this broker
                                    List<KafkaTopicPartitionState<TopicAndPartition>> seedPartitions = new ArrayList<>();
                                    seedPartitions.add(fp);
                                    brokerThread = createAndStartSimpleConsumerThread(seedPartitions, leader, errorHandler);
                                    brokerToThread.put(leader, brokerThread);
                                    newPartitionsQueue = brokerThread.getNewPartitionsQueue(); // update queue for the subsequent partitions
                                }
                            }
                        }
                    }
                }
            }
        }
        catch (InterruptedException e) {
           //......
        }
        finally {
           //......
        }
    }

 

其他一些接口实现，

// ------------------------------------------------------------------------
    //  Kafka 0.8 specific class instantiation
    // ------------------------------------------------------------------------

    @Override
    public TopicAndPartition createKafkaPartitionHandle(KafkaTopicPartition partition) {
        return new TopicAndPartition(partition.getTopic(), partition.getPartition()); //对于kafka0.8，KafkaPartitionHandle就是TopicAndPartition
    }

    // ------------------------------------------------------------------------
    //  Offset handling
    // ------------------------------------------------------------------------

    @Override
    public void commitSpecificOffsetsToKafka(Map<KafkaTopicPartition, Long> offsets) throws Exception {
        ZookeeperOffsetHandler zkHandler = this.zookeeperOffsetHandler;
        if (zkHandler != null) {
            zkHandler.writeOffsets(offsets); //commit offsets是写到zookeeper的
        }
    }

    // ------------------------------------------------------------------------
    //  Utilities
    // ------------------------------------------------------------------------

    private SimpleConsumerThread<T> createAndStartSimpleConsumerThread(
            List<KafkaTopicPartitionState<TopicAndPartition>> seedPartitions,
            Node leader,
            ExceptionProxy errorHandler) throws IOException, ClassNotFoundException
    {
        // each thread needs its own copy of the deserializer, because the deserializer is
        // not necessarily thread safe
        final KeyedDeserializationSchema<T> clonedDeserializer =
                InstantiationUtil.clone(deserializer, userCodeClassLoader);

        // seed thread with list of fetch partitions (otherwise it would shut down immediately again
        SimpleConsumerThread<T> brokerThread = new SimpleConsumerThread<>(
                this, errorHandler, kafkaConfig, leader, seedPartitions, unassignedPartitionsQueue, 
                clonedDeserializer, invalidOffsetBehavior);

        brokerThread.setName(String.format("SimpleConsumer - %s - broker-%s (%s:%d)",
                taskName, leader.id(), leader.host(), leader.port()));
        brokerThread.setDaemon(true);
        brokerThread.start(); //创建和启动SimpleConsumerThread

        LOG.info("Starting thread {}", brokerThread.getName());
        return brokerThread;
    }

 

下面来看看SimpleConsumerThread

class SimpleConsumerThread<T> extends Thread

核心函数run，主要做的是，不停的读取数据的事情，

// these are the actual configuration values of Kafka + their original default values.
    this.soTimeout = getInt(config, "socket.timeout.ms", 30000); //Kafka的一些配置
    this.minBytes = getInt(config, "fetch.min.bytes", 1);
    this.maxWait = getInt(config, "fetch.wait.max.ms", 100);
    this.fetchSize = getInt(config, "fetch.message.max.bytes", 1048576);
    this.bufferSize = getInt(config, "socket.receive.buffer.bytes", 65536);
    this.reconnectLimit = getInt(config, "flink.simple-consumer-reconnectLimit", 3);
   
    // ------------------------------------------------------------------------
    //  main work loop
    // ------------------------------------------------------------------------
    
    @Override
    public void run() {
        try {
            // create the Kafka consumer that we actually use for fetching
            consumer = new SimpleConsumer(broker.host(), broker.port(), soTimeout, bufferSize, clientId); //创建SimpleConsumer
            
            // make sure that all partitions have some offsets to start with
            // those partitions that do not have an offset from a checkpoint need to get
            // their start offset from ZooKeeper
            getMissingOffsetsFromKafka(partitions);  //为没有offset信息的partition，重置offset，从latest或earlist

            // Now, the actual work starts :-)
            int offsetOutOfRangeCount = 0; //用于统计实际执行情况，非法offset，或重连的计数
            int reconnects = 0;
            while (running) {

                // ----------------------------------- partitions list maintenance ----------------------------

                // check queue for new partitions to read from:
                List<KafkaTopicPartitionState<TopicAndPartition>> newPartitions = newPartitionsQueue.pollBatch(); //对于new partitions的处理，主要就是把它们加到partitions当中
                if (newPartitions != null) {
                    // found some new partitions for this thread's broker
                    
                    // check if the new partitions need an offset lookup
                    getMissingOffsetsFromKafka(newPartitions); // 为新的partition重置offset 
                    
                    // add the new partitions (and check they are not already in there)
                    for (KafkaTopicPartitionState<TopicAndPartition> newPartition: newPartitions) {
                        partitions.add(newPartition);
                    }
                }

                if (partitions.size() == 0) { //如果partitions为空，即没有需要消费的partition
                    if (newPartitionsQueue.close()) { //如果此时newPartitionsQueue为closed，那么就不可能会有新的partitions加入，那么该thread就没有存在的意义，不需要继续run
                        // close succeeded. Closing thread
                        running = false; //关闭线程
                        
                        LOG.info("Consumer thread {} does not have any partitions assigned anymore. Stopping thread.", 
                                getName());

                        // add the wake-up marker into the queue to make the main thread
                        // immediately wake up and termination faster
                        unassignedPartitions.add(MARKER);

                        break;
                    } else { //如果newPartitionsQueue没有被关闭，那就等待新的partitions，continue
                        // close failed: fetcher main thread concurrently added new partitions into the queue.
                        // go to top of loop again and get the new partitions
                        continue; 
                    }
                }

                // ----------------------------------- request / response with kafka ----------------------------

                FetchRequestBuilder frb = new FetchRequestBuilder(); //创建FetchRequestBuilder
                frb.clientId(clientId);
                frb.maxWait(maxWait);
                frb.minBytes(minBytes);

                for (KafkaTopicPartitionState<?> partition : partitions) {
                    frb.addFetch(
                            partition.getKafkaTopicPartition().getTopic(),
                            partition.getKafkaTopicPartition().getPartition(),
                            partition.getOffset() + 1, // request the next record
                            fetchSize);
                }
                
                kafka.api.FetchRequest fetchRequest = frb.build(); //创建FetchRequest，一个request可以同时读多个partition，取决于partition和consumer数量的比例

                FetchResponse fetchResponse;
                try {
                    fetchResponse = consumer.fetch(fetchRequest); //从kafka读到数据，包含在FetchResponse中，其中包含从多个partition中取到的数据
                }
                catch (Throwable cce) {
                    //noinspection ConstantConditions
                    if (cce instanceof ClosedChannelException) { //链接kafka异常
                       
                        // we don't know if the broker is overloaded or unavailable.
                        // retry a few times, then return ALL partitions for new leader lookup
                        if (++reconnects >= reconnectLimit) {  //如果达到重连limit，说明确实无法连接到kafka
                            LOG.warn("Unable to reach broker after {} retries. Returning all current partitions", reconnectLimit);
                            for (KafkaTopicPartitionState<TopicAndPartition> fp: this.partitions) {
                                unassignedPartitions.add(fp); //把负责的partitions加入unassignedPartitions，表明这些partition是没人处理的
                            }
                            this.partitions.clear(); //把partitons清空
                            continue; // jump to top of loop: will close thread or subscribe to new partitions //这步会走到上面的partitions.size() == 0的逻辑
                        }
                        try { //如果需要重试，先关闭consumer，然后重新创建consumer，然后continue重试
                            consumer.close();
                        } catch (Throwable t) {
                            LOG.warn("Error while closing consumer connection", t);
                        }
                        // delay & retry
                        Thread.sleep(100);
                        consumer = new SimpleConsumer(broker.host(), broker.port(), soTimeout, bufferSize, clientId);
                        continue; // retry
                    } else {
                        throw cce;
                    }
                }
                reconnects = 0;

                // ---------------------------------------- error handling ----------------------------

                if (fetchResponse == null) {
                    throw new IOException("Fetch from Kafka failed (request returned null)");
                }
                
                if (fetchResponse.hasError()) { //如果fetchResponse有错误
                    String exception = "";
                    List<KafkaTopicPartitionState<TopicAndPartition>> partitionsToGetOffsetsFor = new ArrayList<>();
                    
                    // iterate over partitions to get individual error codes
                    Iterator<KafkaTopicPartitionState<TopicAndPartition>> partitionsIterator = partitions.iterator();
                    boolean partitionsRemoved = false;
                    
                    while (partitionsIterator.hasNext()) {
                        final KafkaTopicPartitionState<TopicAndPartition> fp = partitionsIterator.next();
                        short code = fetchResponse.errorCode(fp.getTopic(), fp.getPartition()); //取得对于该partition的error code

                        if (code == ErrorMapping.OffsetOutOfRangeCode()) { //非法offset，那么需要重新初始化该partition的offset
                            // we were asked to read from an out-of-range-offset (maybe set wrong in Zookeeper)
                            // Kafka's high level consumer is resetting the offset according to 'auto.offset.reset'
                            partitionsToGetOffsetsFor.add(fp);
                        }
                        else if (code == ErrorMapping.NotLeaderForPartitionCode() ||  //如果由于各种不可用，导致无法从该broker上读取到partition的数据
                                code == ErrorMapping.LeaderNotAvailableCode() ||
                                code == ErrorMapping.BrokerNotAvailableCode() ||
                                code == ErrorMapping.UnknownCode())
                        {
                            // the broker we are connected to is not the leader for the partition.
                            LOG.warn("{} is not the leader of {}. Reassigning leader for partition", broker, fp);
                            LOG.debug("Error code = {}", code);

                            unassignedPartitions.add(fp); //那么把该partition放回unassignedPartitions，等待重新分配

                            partitionsIterator.remove(); // unsubscribe the partition ourselves，从当前的partitions列表中把该partition删除
                            partitionsRemoved = true;
                        }
                        else if (code != ErrorMapping.NoError()) {
                            exception += "\nException for " + fp.getTopic() +":"+ fp.getPartition() + ": " +
                                    StringUtils.stringifyException(ErrorMapping.exceptionFor(code));
                        }
                    }
                    if (partitionsToGetOffsetsFor.size() > 0) {
                        // safeguard against an infinite loop.
                        if (offsetOutOfRangeCount++ > 3) { //如果对于partitions，3次重置offset后，offset仍然有非法的，抛异常，防止无限循环
                            throw new RuntimeException("Found invalid offsets more than three times in partitions "
                                    + partitionsToGetOffsetsFor + " Exceptions: " + exception);
                        }
                        // get valid offsets for these partitions and try again.
                        LOG.warn("The following partitions had an invalid offset: {}", partitionsToGetOffsetsFor);
                        getLastOffsetFromKafka(consumer, partitionsToGetOffsetsFor, invalidOffsetBehavior); //重置这些partitions的offset, 根据配置会reset到earliest或latest
                        
                        LOG.warn("The new partition offsets are {}", partitionsToGetOffsetsFor);
                        continue; // jump back to create a new fetch request. The offset has not been touched.
                    }
                    else if (partitionsRemoved) {
                        continue; // create new fetch request
                    }
                    else {
                        // partitions failed on an error
                        throw new IOException("Error while fetching from broker '" + broker +"': " + exception);
                    }
                } else {
                    // successful fetch, reset offsetOutOfRangeCount.
                    offsetOutOfRangeCount = 0;
                }

                // ----------------------------------- process fetch response ----------------------------

                int messagesInFetch = 0;
                int deletedMessages = 0;
                Iterator<KafkaTopicPartitionState<TopicAndPartition>> partitionsIterator = partitions.iterator();
                
                partitionsLoop:
                while (partitionsIterator.hasNext()) {
                    final KafkaTopicPartitionState<TopicAndPartition> currentPartition = partitionsIterator.next();
                    
                    final ByteBufferMessageSet messageSet = fetchResponse.messageSet( //取出fetchResponse关于该partition的数据，封装成ByteBufferMessageSet
                            currentPartition.getTopic(), currentPartition.getPartition());

                    for (MessageAndOffset msg : messageSet) { //对于每天message
                        if (running) {
                            messagesInFetch++;
                            final ByteBuffer payload = msg.message().payload(); //读出message内容
                            final long offset = msg.offset();  //读出message offset
                            
                            if (offset <= currentPartition.getOffset()) { //旧数据，ignore
                                // we have seen this message already
                                LOG.info("Skipping message with offset " + msg.offset()
                                        + " because we have seen messages until (including) "
                                        + currentPartition.getOffset()
                                        + " from topic/partition " + currentPartition.getTopic() + '/'
                                        + currentPartition.getPartition() + " already");
                                continue;
                            }

                            // If the message value is null, this represents a delete command for the message key.
                            // Log this and pass it on to the client who might want to also receive delete messages.
                            byte[] valueBytes;
                            if (payload == null) {
                                deletedMessages++;
                                valueBytes = null;
                            } else {
                                valueBytes = new byte[payload.remaining()];
                                payload.get(valueBytes); //将内容，读入valueBytes
                            }

                            // put key into byte array
                            byte[] keyBytes = null;
                            int keySize = msg.message().keySize();

                            if (keySize >= 0) { // message().hasKey() is doing the same. We save one int deserialization
                                ByteBuffer keyPayload = msg.message().key();
                                keyBytes = new byte[keySize]; //将key读入keyBytes
                                keyPayload.get(keyBytes);
                            }

                            final T value = deserializer.deserialize(keyBytes, valueBytes, //将message反序列化成对象
                                    currentPartition.getTopic(), currentPartition.getPartition(), offset);
                            
                            if (deserializer.isEndOfStream(value)) {
                                // remove partition from subscribed partitions.
                                partitionsIterator.remove();
                                continue partitionsLoop;
                            }
                            
                            owner.emitRecord(value, currentPartition, offset); //emit 数据
                        }
                        else {
                            // no longer running
                            return;
                        }
                    }
                }
            } // end of fetch loop

        }
    }

 

最后，看看

FlinkKafkaConsumerBase

/**
 * Base class of all Flink Kafka Consumer data sources.
 * This implements the common behavior across all Kafka versions.
 * 
 * <p>The Kafka version specific behavior is defined mainly in the specific subclasses of the
 * {@link AbstractFetcher}.
 * 
 * @param <T> The type of records produced by this data source
 */
public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFunction<T> implements 
        CheckpointListener,
        CheckpointedAsynchronously<HashMap<KafkaTopicPartition, Long>>,
        ResultTypeQueryable<T>

这个是对所有版本kafka的抽象，

@Override
    public void run(SourceContext<T> sourceContext) throws Exception {
        
        // figure out which partitions this subtask should process
        final List<KafkaTopicPartition> thisSubtaskPartitions = assignPartitions(allSubscribedPartitions, //对应于topic partition数和consumer数，一个consumer应该分配哪些partitions，这里逻辑就是简单的取模
                getRuntimeContext().getNumberOfParallelSubtasks(), getRuntimeContext().getIndexOfThisSubtask());
        
        // we need only do work, if we actually have partitions assigned
        if (!thisSubtaskPartitions.isEmpty()) {

            // (1) create the fetcher that will communicate with the Kafka brokers
            final AbstractFetcher<T, ?> fetcher = createFetcher(  //创建Fetcher
                    sourceContext, thisSubtaskPartitions, 
                    periodicWatermarkAssigner, punctuatedWatermarkAssigner,
                    (StreamingRuntimeContext) getRuntimeContext());

            // (2) set the fetcher to the restored checkpoint offsets
            if (restoreToOffset != null) {  //这个如果从checkpoint中读出offset状态
                fetcher.restoreOffsets(restoreToOffset); //恢复offset
            }

            // publish the reference, for snapshot-, commit-, and cancel calls
            // IMPORTANT: We can only do that now, because only now will calls to
            //            the fetchers 'snapshotCurrentState()' method return at least
            //            the restored offsets
            this.kafkaFetcher = fetcher;
            if (!running) {
                return;
            }
            
            // (3) run the fetcher' main work method
            fetcher.runFetchLoop();  //开始run fetcher
        }
    }

    @Override
    public HashMap<KafkaTopicPartition, Long> snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
        
        final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;

        HashMap<KafkaTopicPartition, Long> currentOffsets = fetcher.snapshotCurrentState(); //snapshot当前的offset

        // the map cannot be asynchronously updated, because only one checkpoint call can happen
        // on this function at a time: either snapshotState() or notifyCheckpointComplete()
        pendingCheckpoints.put(checkpointId, currentOffsets); //cache当前的checkpointid，等待该checkpoint完成
        
        // truncate the map, to prevent infinite growth
        while (pendingCheckpoints.size() > MAX_NUM_PENDING_CHECKPOINTS) { //删除过期的，或老的checkpoints
            pendingCheckpoints.remove(0);
        }

        return currentOffsets;
    }

    @Override
    public void restoreState(HashMap<KafkaTopicPartition, Long> restoredOffsets) {
        LOG.info("Setting restore state in the FlinkKafkaConsumer");
        restoreToOffset = restoredOffsets;
    }

    @Override
    public void notifyCheckpointComplete(long checkpointId) throws Exception { //当这个checkpoint完成时，需要通知kafkaconsumer，这个时候才会真正的commit offset

        final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;

        try {
            final int posInMap = pendingCheckpoints.indexOf(checkpointId);

            @SuppressWarnings("unchecked")
            HashMap<KafkaTopicPartition, Long> checkpointOffsets = 
                    (HashMap<KafkaTopicPartition, Long>) pendingCheckpoints.remove(posInMap);

            // remove older checkpoints in map
            for (int i = 0; i < posInMap; i++) { //比该checkpoint老的未完成的checkpoint已经没有意义，删除
                pendingCheckpoints.remove(0);
            }

            fetcher.commitSpecificOffsetsToKafka(checkpointOffsets); //真正的commit offset，这个是通用接口，虽然对于kafka0.8，Fetcher里面本身也是会定期提交的，checkpoint一般秒级别比定期提交更频繁些
        }
    }
    
    /**
     * Selects which of the given partitions should be handled by a specific consumer,
     * given a certain number of consumers.
     * 
     * @param allPartitions The partitions to select from
     * @param numConsumers The number of consumers
     * @param consumerIndex The index of the specific consumer
     * 
     * @return The sublist of partitions to be handled by that consumer.
     */
    protected static List<KafkaTopicPartition> assignPartitions(
            List<KafkaTopicPartition> allPartitions,
            int numConsumers, int consumerIndex)
    {
        final List<KafkaTopicPartition> thisSubtaskPartitions = new ArrayList<>(
                allPartitions.size() / numConsumers + 1);

        for (int i = 0; i < allPartitions.size(); i++) {
            if (i % numConsumers == consumerIndex) {
                thisSubtaskPartitions.add(allPartitions.get(i));
            }
        }
        
        return thisSubtaskPartitions;
    }

 

针对kafka0.8的consumer

public class FlinkKafkaConsumer08<T> extends FlinkKafkaConsumerBase<T> {

    /**
     * Creates a new Kafka streaming source consumer for Kafka 0.8.x
     *
     * This constructor allows passing multiple topics and a key/value deserialization schema.
     * 
     * @param topics
     *           The Kafka topics to read from.
     * @param deserializer
     *           The keyed de-/serializer used to convert between Kafka's byte messages and Flink's objects.
     * @param props
     *           The properties that are used to configure both the fetcher and the offset handler.
     */
    public FlinkKafkaConsumer08(List<String> topics, KeyedDeserializationSchema<T> deserializer, Properties props) {
        super(deserializer);

        // validate the zookeeper properties
        validateZooKeeperConfig(props);

        this.invalidOffsetBehavior = getInvalidOffsetBehavior(props); //当offset非法的时候，选择从哪里重置，这里支持earlist或latest
        this.autoCommitInterval = PropertiesUtil.getLong(props, "auto.commit.interval.ms", 60000); //offset commit的间隔，默认是1分钟

        // Connect to a broker to get the partitions for all topics
        List<KafkaTopicPartition> partitionInfos = 
                KafkaTopicPartition.dropLeaderData(getPartitionsForTopic(topics, props)); //这里只是取出topic相关的partition的信息

        setSubscribedPartitions(partitionInfos); //将这部分，即该consumer消费的partitions，加入到SubscribedPartitions，表明这些已经有consumer消费了
    }
    
    /**
     * Send request to Kafka to get partitions for topic.
     * 
     * @param topics The name of the topics.
     * @param properties The properties for the Kafka Consumer that is used to query the partitions for the topic. 
     */
    public static List<KafkaTopicPartitionLeader> getPartitionsForTopic(List<String> topics, Properties properties) { //这里的逻辑是如果从kafka取得topic的partititon信息，这里配置配的是broker list，而非zk，所以他每次会随机从所有的brokers中挑一个去读取partitions信息
        String seedBrokersConfString = properties.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG);
        final int numRetries = getInt(properties, GET_PARTITIONS_RETRIES_KEY, DEFAULT_GET_PARTITIONS_RETRIES);

        checkNotNull(seedBrokersConfString, "Configuration property %s not set", ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG);
        String[] seedBrokers = seedBrokersConfString.split(",");
        List<KafkaTopicPartitionLeader> partitions = new ArrayList<>();

        final String clientId = "flink-kafka-consumer-partition-lookup";
        final int soTimeout = getInt(properties, "socket.timeout.ms", 30000);
        final int bufferSize = getInt(properties, "socket.receive.buffer.bytes", 65536);

        Random rnd = new Random();
        retryLoop: for (int retry = 0; retry < numRetries; retry++) {
            // we pick a seed broker randomly to avoid overloading the first broker with all the requests when the
            // parallel source instances start. Still, we try all available brokers.
            int index = rnd.nextInt(seedBrokers.length);
            brokersLoop: for (int arrIdx = 0; arrIdx < seedBrokers.length; arrIdx++) {
                String seedBroker = seedBrokers[index];
                LOG.info("Trying to get topic metadata from broker {} in try {}/{}", seedBroker, retry, numRetries);
                if (++index == seedBrokers.length) {
                    index = 0;
                }

                URL brokerUrl = NetUtils.getCorrectHostnamePort(seedBroker);
                SimpleConsumer consumer = null;
                try {
                    consumer = new SimpleConsumer(brokerUrl.getHost(), brokerUrl.getPort(), soTimeout, bufferSize, clientId);

                    TopicMetadataRequest req = new TopicMetadataRequest(topics);
                    kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);

                    List<TopicMetadata> metaData = resp.topicsMetadata();

                    // clear in case we have an incomplete list from previous tries
                    partitions.clear();
                    for (TopicMetadata item : metaData) {
                        if (item.errorCode() != ErrorMapping.NoError()) {
                            // warn and try more brokers
                            LOG.warn("Error while getting metadata from broker " + seedBroker + " to find partitions " +
                                    "for " + topics.toString() + ". Error: " + ErrorMapping.exceptionFor(item.errorCode()).getMessage());
                            continue brokersLoop;
                        }
                        if (!topics.contains(item.topic())) {
                            LOG.warn("Received metadata from topic " + item.topic() + " even though it was not requested. Skipping ...");
                            continue brokersLoop;
                        }
                        for (PartitionMetadata part : item.partitionsMetadata()) {
                            Node leader = brokerToNode(part.leader());
                            KafkaTopicPartition ktp = new KafkaTopicPartition(item.topic(), part.partitionId());
                            KafkaTopicPartitionLeader pInfo = new KafkaTopicPartitionLeader(ktp, leader);
                            partitions.add(pInfo);
                        }
                    }
                    break retryLoop; // leave the loop through the brokers
                } catch (Exception e) {
                    LOG.warn("Error communicating with broker " + seedBroker + " to find partitions for " + topics.toString() + "." +
                            "" + e.getClass() + ". Message: " + e.getMessage());
                    LOG.debug("Detailed trace", e);
                    // we sleep a bit. Retrying immediately doesn't make sense in cases where Kafka is reorganizing the leader metadata
                    try {
                        Thread.sleep(500);
                    } catch (InterruptedException e1) {
                        // sleep shorter.
                    }
                } finally {
                    if (consumer != null) {
                        consumer.close();
                    }
                }
            } // brokers loop
        } // retries loop
        return partitions;
    }

    private static long getInvalidOffsetBehavior(Properties config) {
        final String val = config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "largest");
        if (val.equals("none")) {
            throw new IllegalArgumentException("Cannot use '" + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG
                    + "' value 'none'. Possible values: 'latest', 'largest', or 'earliest'.");
        }
        else if (val.equals("largest") || val.equals("latest")) { // largest is kafka 0.8, latest is kafka 0.9
            return OffsetRequest.LatestTime();
        } else {
            return OffsetRequest.EarliestTime();
        }
    }