WebRuntimeMonitor
.GET("/jobs/:jobid/vertices/:vertexid/metrics", handler(new JobVertexMetricsHandler(metricFetcher)))
.GET("/jobs/:jobid/metrics", handler(new JobMetricsHandler(metricFetcher)))
.GET("/taskmanagers/:" + TaskManagersHandler.TASK_MANAGER_ID_KEY + "/metrics", handler(new TaskManagerMetricsHandler(metricFetcher)))
.GET("/jobmanager/metrics", handler(new JobManagerMetricsHandler(metricFetcher)))
JobVertexMetricsHandler
AbstractMetricsHandler
MetricFetcher
核心就是fetchMetrics函数，会从JobManager获取数据，
private void fetchMetrics() { try { Option<scala.Tuple2<ActorGateway, Integer>> jobManagerGatewayAndWebPort = retriever.getJobManagerGatewayAndWebPort(); if (jobManagerGatewayAndWebPort.isDefined()) { ActorGateway jobManager = jobManagerGatewayAndWebPort.get()._1(); //得到JobManager的ActorGateway /** * Remove all metrics that belong to a job that is not running and no longer archived. */ Future<Object> jobDetailsFuture = jobManager.ask(new RequestJobDetails(true, true), timeout); //生成request获取job状态 jobDetailsFuture .onSuccess(new OnSuccess<Object>() { @Override public void onSuccess(Object result) throws Throwable { MultipleJobsDetails details = (MultipleJobsDetails) result; ArrayList<String> toRetain = new ArrayList<>(); for (JobDetails job : details.getRunningJobs()) { toRetain.add(job.getJobId().toString()); } for (JobDetails job : details.getFinishedJobs()) { toRetain.add(job.getJobId().toString()); } synchronized (metrics) { metrics.jobs.keySet().retainAll(toRetain); //只保留Runing和Finished的job，即不正常的都删掉 } } }, ctx); logErrorOnFailure(jobDetailsFuture, "Fetching of JobDetails failed."); String jobManagerPath = jobManager.path(); String queryServicePath = jobManagerPath.substring(0, jobManagerPath.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME; ActorRef jobManagerQueryService = actorSystem.actorFor(queryServicePath); queryMetrics(jobManagerQueryService); //查询jobManager的Metrics /** * We first request the list of all registered task managers from the job manager, and then * request the respective metric dump from each task manager. * * All stored metrics that do not belong to a registered task manager will be removed. */ Future<Object> registeredTaskManagersFuture = jobManager.ask(JobManagerMessages.getRequestRegisteredTaskManagers(), timeout); //查询所有taskManager registeredTaskManagersFuture .onSuccess(new OnSuccess<Object>() { @Override public void onSuccess(Object result) throws Throwable { Iterable<Instance> taskManagers = ((JobManagerMessages.RegisteredTaskManagers) result).asJavaIterable(); List<String> activeTaskManagers = new ArrayList<>(); for (Instance taskManager : taskManagers) { //遍历taskManager activeTaskManagers.add(taskManager.getId().toString()); String taskManagerPath = taskManager.getTaskManagerGateway().getAddress(); String queryServicePath = taskManagerPath.substring(0, taskManagerPath.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME + "_" + taskManager.getTaskManagerID().getResourceIdString(); ActorRef taskManagerQueryService = actorSystem.actorFor(queryServicePath); queryMetrics(taskManagerQueryService); //查询每个taskMananger的metrics } synchronized (metrics) { // remove all metrics belonging to unregistered task managers metrics.taskManagers.keySet().retainAll(activeTaskManagers); //删除所有的未注册的TaskManager } } }, ctx); logErrorOnFailure(registeredTaskManagersFuture, "Fetchin list of registered TaskManagers failed."); } } catch (Exception e) { LOG.warn("Exception while fetching metrics.", e); } }
queryMetrics
/** * Requests a metric dump from the given actor. * * @param actor ActorRef to request the dump from */ private void queryMetrics(ActorRef actor) { Future<Object> metricQueryFuture = new BasicGateway(actor).ask(MetricQueryService.getCreateDump(), timeout); //获取metrics dump metricQueryFuture .onSuccess(new OnSuccess<Object>() { @Override public void onSuccess(Object result) throws Throwable { addMetrics(result); } }, ctx); logErrorOnFailure(metricQueryFuture, "Fetching metrics failed."); } private void addMetrics(Object result) throws IOException { byte[] data = (byte[]) result; List<MetricDump> dumpedMetrics = deserializer.deserialize(data); for (MetricDump metric : dumpedMetrics) { metrics.add(metric); //把metrics dump加入metrics store } }
MetricStore
用嵌套的hashmap来存储metrics，瞬时值
final JobManagerMetricStore jobManager = new JobManagerMetricStore(); final Map<String, TaskManagerMetricStore> taskManagers = new HashMap<>(); final Map<String, JobMetricStore> jobs = new HashMap<>();
public static class JobManagerMetricStore extends ComponentMetricStore { } private static abstract class ComponentMetricStore { public final Map<String, String> metrics = new HashMap<>(); //store就是一个map public String getMetric(String name, String defaultValue) { String value = this.metrics.get(name); return value != null ? value : defaultValue; } }
MetricQueryService
public class MetricQueryService extends UntypedActor { private static final Logger LOG = LoggerFactory.getLogger(MetricQueryService.class); public static final String METRIC_QUERY_SERVICE_NAME = "MetricQueryService"; private static final CharacterFilter FILTER = new CharacterFilter() { @Override public String filterCharacters(String input) { return replaceInvalidChars(input); } }; private final MetricDumpSerializer serializer = new MetricDumpSerializer(); private final Map<Gauge<?>, Tuple2<QueryScopeInfo, String>> gauges = new HashMap<>(); private final Map<Counter, Tuple2<QueryScopeInfo, String>> counters = new HashMap<>(); private final Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms = new HashMap<>(); private final Map<Meter, Tuple2<QueryScopeInfo, String>> meters = new HashMap<>();
收到CreateDump请求，
} else if (message instanceof CreateDump) { byte[] dump = serializer.serialize(counters, gauges, histograms, meters); getSender().tell(dump, getSelf());
Start
/** * Starts the MetricQueryService actor in the given actor system. * * @param actorSystem The actor system running the MetricQueryService * @param resourceID resource ID to disambiguate the actor name * @return actor reference to the MetricQueryService */ public static ActorRef startMetricQueryService(ActorSystem actorSystem, ResourceID resourceID) { String actorName = resourceID == null ? METRIC_QUERY_SERVICE_NAME : METRIC_QUERY_SERVICE_NAME + "_" + resourceID.getResourceIdString(); return actorSystem.actorOf(Props.create(MetricQueryService.class), actorName); }
在MetricRegistry中把metrics注册到QueryService中，
if (queryService != null) { MetricQueryService.notifyOfAddedMetric(queryService, metric, metricName, group); }
采集点
numRecordsIn
StreamInputProcessor –> processInput
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
public boolean processInput(OneInputStreamOperator<IN, ?> streamOperator, final Object lock) throws Exception {
if (numRecordsIn == null) {
numRecordsIn = ((OperatorMetricGroup) streamOperator.getMetricGroup()).getIOMetricGroup().getNumRecordsInCounter();
}
//......
// now we can do the actual processing
StreamRecord<IN> record = recordOrMark.asRecord();
synchronized (lock) {
numRecordsIn.inc(); //执行processElement前加一
streamOperator.setKeyContextElement1(record);
streamOperator.processElement(record);
}
return true;
如果是chaining，
ChainingOutput
private static class ChainingOutput<T> implements Output<StreamRecord<T>> { protected final OneInputStreamOperator<T, ?> operator; protected final Counter numRecordsIn; public ChainingOutput(OneInputStreamOperator<T, ?> operator) { this.operator = operator; this.numRecordsIn = ((OperatorMetricGroup) operator.getMetricGroup()).getIOMetricGroup().getNumRecordsInCounter(); //初始化 } @Override public void collect(StreamRecord<T> record) { try { numRecordsIn.inc(); //对于chain，在output时调用processElement operator.setKeyContextElement1(record); operator.processElement(record); } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } }
numRecordsOut
在AbstractStreamOperator初始化时，
生成CountingOutput
@Override public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) { this.container = containingTask; this.config = config; this.metrics = container.getEnvironment().getMetricGroup().addOperator(config.getOperatorName()); this.output = new CountingOutput(output, ((OperatorMetricGroup) this.metrics).getIOMetricGroup().getNumRecordsOutCounter()); //生成CountingOutput
这个output，
在processWatermark，processElement中会用于emit数据
output.emitWatermark(mark);
public class CountingOutput implements Output<StreamRecord<OUT>> { private final Output<StreamRecord<OUT>> output; private final Counter numRecordsOut; public CountingOutput(Output<StreamRecord<OUT>> output, Counter counter) { this.output = output; this.numRecordsOut = counter; } @Override public void emitWatermark(Watermark mark) { output.emitWatermark(mark); } @Override public void emitLatencyMarker(LatencyMarker latencyMarker) { output.emitLatencyMarker(latencyMarker); } @Override public void collect(StreamRecord<OUT> record) { numRecordsOut.inc(); //发出的时候，inc numRecordsOut output.collect(record); } @Override public void close() { output.close(); } }
注意numRecordsOut和numRecordsIn，除了会统计operator级别的，还会统计task级别的，逻辑在
AbstractStreamOperator
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) { this.container = containingTask; this.config = config; this.metrics = container.getEnvironment().getMetricGroup().addOperator(config.getOperatorName()); this.output = new CountingOutput(output, ((OperatorMetricGroup) this.metrics).getIOMetricGroup().getNumRecordsOutCounter()); if (config.isChainStart()) { ((OperatorMetricGroup) this.metrics).getIOMetricGroup().reuseInputMetricsForTask(); } if (config.isChainEnd()) { ((OperatorMetricGroup) this.metrics).getIOMetricGroup().reuseOutputMetricsForTask(); }
OperatorIOMetricGroup
public void reuseInputMetricsForTask() { TaskIOMetricGroup taskIO = parentMetricGroup.parent().getIOMetricGroup(); taskIO.reuseRecordsInputCounter(this.numRecordsIn); } public void reuseOutputMetricsForTask() { TaskIOMetricGroup taskIO = parentMetricGroup.parent().getIOMetricGroup(); taskIO.reuseRecordsOutputCounter(this.numRecordsOut); }
可以看到，会将ChainHead的numRecordsIn，set到task的TaskIOMetricGroup
而将ChainEnd的numRecordsOut，set到task的TaskIOMetricGroup
看起来很合理
numRecordInPerSecond，numRecordsOutPerSecond
在OperatorIOMetricGroup
public OperatorIOMetricGroup(OperatorMetricGroup parentMetricGroup) { super(parentMetricGroup); numRecordsIn = parentMetricGroup.counter(MetricNames.IO_NUM_RECORDS_IN); numRecordsOut = parentMetricGroup.counter(MetricNames.IO_NUM_RECORDS_OUT); numRecordsInRate = parentMetricGroup.meter(MetricNames.IO_NUM_RECORDS_IN_RATE, new MeterView(numRecordsIn, 60)); numRecordsOutRate = parentMetricGroup.meter(MetricNames.IO_NUM_RECORDS_OUT_RATE, new MeterView(numRecordsOut, 60)); }
可以看到numRecordsInRate和numRecordsOutRate，只是numRecordsIn和numRecordsOut的MeterView
public class MeterView implements Meter, View { /** The underlying counter maintaining the count */ private final Counter counter; /** The time-span over which the average is calculated */ private final int timeSpanInSeconds; /** Circular array containing the history of values */ private final long[] values; /** The index in the array for the current time */ private int time = 0; /** The last rate we computed */ private double currentRate = 0; public MeterView(Counter counter, int timeSpanInSeconds) { this.counter = counter; this.timeSpanInSeconds = timeSpanInSeconds - (timeSpanInSeconds % UPDATE_INTERVAL_SECONDS); //timeSpanInSeconds需要是UPDATE_INTERVAL_SECONDS(5)的倍数， this.values = new long[this.timeSpanInSeconds / UPDATE_INTERVAL_SECONDS + 1]; //比如timeSpanInSeconds为60，那么就需要保存12个value } @Override public void markEvent() { this.counter.inc(); } @Override public void markEvent(long n) { this.counter.inc(n); } @Override public long getCount() { return counter.getCount(); } @Override public double getRate() { //获取平均值 return currentRate; } @Override public void update() { //会被以UPDATE_INTERVAL_SECONDS为间隔调用 time = (time + 1) % values.length; values[time] = counter.getCount(); currentRate = ((double) (values[time] - values[(time + 1) % values.length]) / timeSpanInSeconds); //values保存了timeSpanInSeconds时间段的counter的变化过程，所以用最新的减最老的，再除以timeSpanInSeconds } }
这个实现真是tricky，不好的设计
在MetricRegistry中，会创建
ViewUpdater
public void register(Metric metric, String metricName, AbstractMetricGroup group) { try { if (reporters != null) { for (int i = 0; i < reporters.size(); i++) { MetricReporter reporter = reporters.get(i); if (reporter != null) { FrontMetricGroup front = new FrontMetricGroup<AbstractMetricGroup<?>>(i, group); reporter.notifyOfAddedMetric(metric, metricName, front); } } } if (queryService != null) { MetricQueryService.notifyOfAddedMetric(queryService, metric, metricName, group); } if (metric instanceof View) { if (viewUpdater == null) { viewUpdater = new ViewUpdater(executor); } viewUpdater.notifyOfAddedView((View) metric); } } catch (Exception e) { LOG.error("Error while registering metric.", e); } }
并且在register metrics的时候，除了注册到reporter，MetricQueryService
如果是view的子类还要，注册到ViewUpdater
public ViewUpdater(ScheduledExecutorService executor) { executor.scheduleWithFixedDelay(new ViewUpdaterTask(lock, toAdd, toRemove), 5, UPDATE_INTERVAL_SECONDS, TimeUnit.SECONDS); }
ViewUpdater会定期执行ViewUpdaterTask，task中就会调用view的update
numBytesInLocal, numBytesInRemote
在RemoteInputChannel和LocalInputChannel中，
public LocalInputChannel( SingleInputGate inputGate, int channelIndex, ResultPartitionID partitionId, ResultPartitionManager partitionManager, TaskEventDispatcher taskEventDispatcher, int initialBackoff, int maxBackoff, TaskIOMetricGroup metrics) { super(inputGate, channelIndex, partitionId, initialBackoff, maxBackoff, metrics.getNumBytesInLocalCounter()); //metrics.getNumBytesInLocalCounter() public RemoteInputChannel( SingleInputGate inputGate, int channelIndex, ResultPartitionID partitionId, ConnectionID connectionId, ConnectionManager connectionManager, int initialBackOff, int maxBackoff, TaskIOMetricGroup metrics) { super(inputGate, channelIndex, partitionId, initialBackOff, maxBackoff, metrics.getNumBytesInRemoteCounter()); // metrics.getNumBytesInRemoteCounter()
并且都会在
BufferAndAvailability getNextBuffer()
会调用，
numBytesIn.inc(next.getSize());
numBytesOut
RecordWriter
public class RecordWriter<T extends IOReadableWritable> { private Counter numBytesOut = new SimpleCounter(); public void emit(T record) throws IOException, InterruptedException { for (int targetChannel : channelSelector.selectChannels(record, numChannels)) { sendToTarget(record, targetChannel); } } private void sendToTarget(T record, int targetChannel) throws IOException, InterruptedException { RecordSerializer<T> serializer = serializers[targetChannel]; synchronized (serializer) { SerializationResult result = serializer.addRecord(record); while (result.isFullBuffer()) { Buffer buffer = serializer.getCurrentBuffer(); if (buffer != null) { numBytesOut.inc(buffer.getSize()); //计数numBytesOut writeAndClearBuffer(buffer, targetChannel, serializer); // If this was a full record, we are done. Not breaking // out of the loop at this point will lead to another // buffer request before breaking out (that would not be // a problem per se, but it can lead to stalls in the // pipeline). if (result.isFullRecord()) { break; } } else { buffer = targetPartition.getBufferProvider().requestBufferBlocking(); result = serializer.setNextBuffer(buffer); } } } }
RecordWriterOutput.collect –> StreamRecordWriter.emit –> RecordWriter.emit
inputQueueLength, outputQueueLength, inPoolUsage, outPoolUsage
TaskIOMetricGroup
/** * Initialize Buffer Metrics for a task */ public void initializeBufferMetrics(Task task) { final MetricGroup buffers = addGroup("buffers"); buffers.gauge("inputQueueLength", new InputBuffersGauge(task)); buffers.gauge("outputQueueLength", new OutputBuffersGauge(task)); buffers.gauge("inPoolUsage", new InputBufferPoolUsageGauge(task)); buffers.gauge("outPoolUsage", new OutputBufferPoolUsageGauge(task)); }
inputQueueLength
for (SingleInputGate inputGate : task.getAllInputGates()) { totalBuffers += inputGate.getNumberOfQueuedBuffers(); }
inputGate.getNumberOfQueuedBuffers
for (InputChannel channel : inputChannels.values()) { if (channel instanceof RemoteInputChannel) { // 只统计RemoteInputChannel totalBuffers += ((RemoteInputChannel) channel).getNumberOfQueuedBuffers(); } }
getNumberOfQueuedBuffers
/** * The received buffers. Received buffers are enqueued by the network I/O thread and the queue * is consumed by the receiving task thread. */ private final Queue<Buffer> receivedBuffers = new ArrayDeque<>(); public int getNumberOfQueuedBuffers() { synchronized (receivedBuffers) { return receivedBuffers.size(); } }
outputQueueLength
for (ResultPartition producedPartition : task.getProducedPartitions()) { totalBuffers += producedPartition.getNumberOfQueuedBuffers(); }
ResultPartition getNumberOfQueuedBuffers
for (ResultSubpartition subpartition : subpartitions) { totalBuffers += subpartition.getNumberOfQueuedBuffers(); }
SpillableSubpartition getNumberOfQueuedBuffers
class SpillableSubpartition extends ResultSubpartition { /** Buffers are kept in this queue as long as we weren't ask to release any. */ private final ArrayDeque<Buffer> buffers = new ArrayDeque<>(); @Override public int getNumberOfQueuedBuffers() { return buffers.size(); }
inputQueueLength, outputQueueLength
指标的含义是，inputchannel和resultparitition，持有的buffer个数，这些buffer被读完后会release，所以链路通畅的话，length应该会很小
inPoolUsage
int usedBuffers = 0; int bufferPoolSize = 0; for (SingleInputGate inputGate : task.getAllInputGates()) { usedBuffers += inputGate.getBufferPool().bestEffortGetNumOfUsedBuffers(); bufferPoolSize += inputGate.getBufferPool().getNumBuffers(); } if (bufferPoolSize != 0) { return ((float) usedBuffers) / bufferPoolSize; } else { return 0.0f; }
bestEffortGetNumOfUsedBuffers()
@Override public int bestEffortGetNumOfUsedBuffers() { return Math.max(0, numberOfRequestedMemorySegments - availableMemorySegments.size()); }
numberOfRequestedMemorySegments，从bufferpool申请多少
availableMemorySegments，可用的
所以相减就是使用多少
outPoolUsage
int usedBuffers = 0; int bufferPoolSize = 0; for (ResultPartition resultPartition : task.getProducedPartitions()) { usedBuffers += resultPartition.getBufferPool().bestEffortGetNumOfUsedBuffers(); bufferPoolSize += resultPartition.getBufferPool().getNumBuffers(); } if (bufferPoolSize != 0) { return ((float) usedBuffers) / bufferPoolSize; } else { return 0.0f; }
和inPoolUsage类似，也是看bufferPool的情况
所以inPoolUsage，outPoolUsage表示的是inputgate和resultpartition中bufferpool的使用情况
这个bufferpool是inputgate初始化的时候，注册到NetworkEnvironment创建的，
// Setup the buffer pool for each buffer reader final SingleInputGate[] inputGates = task.getAllInputGates(); for (SingleInputGate gate : inputGates) { BufferPool bufferPool = null; try { bufferPool = networkBufferPool.createBufferPool(gate.getNumberOfInputChannels(), false); gate.setBufferPool(bufferPool); }
可以看到默认大小是，inputchanels的size
如果pool用完了，那么inputGate和ResultPartiton就无法继续读取新的数据
latency
在AbstractStreamOperator中，
setup，
protected LatencyGauge latencyGauge;
latencyGauge = this.metrics.gauge("latency", new LatencyGauge(historySize));
注意，这里metrics是OperatorMetricGroup
this.metrics = container.getEnvironment().getMetricGroup().addOperator(config.getOperatorName());
TaskMetricGroup
public OperatorMetricGroup addOperator(String name) { OperatorMetricGroup operator = new OperatorMetricGroup(this.registry, this, name); synchronized (this) { OperatorMetricGroup previous = operators.put(name, operator); if (previous == null) { // no operator group so far return operator; } else { // already had an operator group. restore that one. operators.put(name, previous); return previous; } } }
LatencyGauge的定义，
/** * The gauge uses a HashMap internally to avoid classloading issues when accessing * the values using JMX. */ protected static class LatencyGauge implements Gauge<Map<String, HashMap<String, Double>>> { //LatencySourceDescriptor，包含vertexID和subtaskIndex //DescriptiveStatistics，统计模块 private final Map<LatencySourceDescriptor, DescriptiveStatistics> latencyStats = new HashMap<>(); private final int historySize; LatencyGauge(int historySize) { this.historySize = historySize; } public void reportLatency(LatencyMarker marker, boolean isSink) { LatencySourceDescriptor sourceDescriptor = LatencySourceDescriptor.of(marker, !isSink); DescriptiveStatistics sourceStats = latencyStats.get(sourceDescriptor); if (sourceStats == null) { //初始化DescriptiveStatistics // 512 element window (4 kb) sourceStats = new DescriptiveStatistics(this.historySize); latencyStats.put(sourceDescriptor, sourceStats); } long now = System.currentTimeMillis(); sourceStats.addValue(now - marker.getMarkedTime()); //当前时间和source发出时时间差值作为延迟 } @Override public Map<String, HashMap<String, Double>> getValue() { while (true) { try { Map<String, HashMap<String, Double>> ret = new HashMap<>(); for (Map.Entry<LatencySourceDescriptor, DescriptiveStatistics> source : latencyStats.entrySet()) { HashMap<String, Double> sourceStatistics = new HashMap<>(6); sourceStatistics.put("max", source.getValue().getMax()); sourceStatistics.put("mean", source.getValue().getMean()); sourceStatistics.put("min", source.getValue().getMin()); sourceStatistics.put("p50", source.getValue().getPercentile(50)); sourceStatistics.put("p95", source.getValue().getPercentile(95)); sourceStatistics.put("p99", source.getValue().getPercentile(99)); ret.put(source.getKey().toString(), sourceStatistics); } return ret; // Concurrent access onto the "latencyStats" map could cause // ConcurrentModificationExceptions. To avoid unnecessary blocking // of the reportLatency() method, we retry this operation until // it succeeds. } catch(ConcurrentModificationException ignore) { LOG.debug("Unable to report latency statistics", ignore); } } } }
这个Gauge.getValue返回的是个map，太奇葩
latencyStats里面有多少entry，取决于有多少source，以及每个source有几个并发
因为他要记录，每个source operator的某个subtask，到当前operator的该subtask的延迟
public static LatencySourceDescriptor of(LatencyMarker marker, boolean ignoreSubtaskIndex) { if (ignoreSubtaskIndex) { return new LatencySourceDescriptor(marker.getVertexID(), -1); } else { return new LatencySourceDescriptor(marker.getVertexID(), marker.getSubtaskIndex()); } }
LatencySourceDescriptor构造函数，由vertexid，和subtaskIndex组成
如果忽略subtaskindex，置为-1
流程
StreamSource
定义LatencyMarksEmitter
private static class LatencyMarksEmitter<OUT> { private final ScheduledFuture<?> latencyMarkTimer; public LatencyMarksEmitter( final ProcessingTimeService processingTimeService, final Output<StreamRecord<OUT>> output, long latencyTrackingInterval, final int vertexID, final int subtaskIndex) { latencyMarkTimer = processingTimeService.scheduleAtFixedRate( //根据processingTime定期发送latencyMarker new ProcessingTimeCallback() { @Override public void onProcessingTime(long timestamp) throws Exception { try { // ProcessingTimeService callbacks are executed under the checkpointing lock output.emitLatencyMarker(new LatencyMarker(timestamp, vertexID, subtaskIndex)); //emitLatencyMarker，以processTime为初始时间 } catch (Throwable t) { // we catch the Throwables here so that we don't trigger the processing // timer services async exception handler LOG.warn("Error while emitting latency marker.", t); } } }, 0L, latencyTrackingInterval); }
source.run，当isLatencyTrackingEnabled，schedule latency marker
public void run(final Object lockingObject, final Output<StreamRecord<OUT>> collector) throws Exception { final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic(); LatencyMarksEmitter latencyEmitter = null; if(getExecutionConfig().isLatencyTrackingEnabled()) { latencyEmitter = new LatencyMarksEmitter<>( getProcessingTimeService(), collector, getExecutionConfig().getLatencyTrackingInterval(), getOperatorConfig().getVertexID(), getRuntimeContext().getIndexOfThisSubtask()); }
StreamInputProcessor –> processInput
如果是isLatencyMarker
else if(recordOrMark.isLatencyMarker()) { // handle latency marker synchronized (lock) { streamOperator.processLatencyMarker(recordOrMark.asLatencyMarker()); } continue; }
对于，chaining， ChainingOutput
private static class ChainingOutput<T> implements Output<StreamRecord<T>> { protected final OneInputStreamOperator<T, ?> operator; protected final Counter numRecordsIn; @Override public void emitLatencyMarker(LatencyMarker latencyMarker) { try { operator.processLatencyMarker(latencyMarker); } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } }
AbstractStreamOperator
public void processLatencyMarker(LatencyMarker latencyMarker) throws Exception { reportOrForwardLatencyMarker(latencyMarker); }
protected void reportOrForwardLatencyMarker(LatencyMarker marker) { // all operators are tracking latencies this.latencyGauge.reportLatency(marker, false); // everything except sinks forwards latency markers this.output.emitLatencyMarker(marker); }
调用到latencyGauge.reportLatency，逻辑如上
后续继续emitLatencyMarker
currentLowWatermark, checkpointAlignmentTime
OneInputStreamTask
@Override public void init() throws Exception { if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<IN>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration()); // make sure that stream tasks report their I/O statistics inputProcessor.setMetricGroup(getEnvironment().getMetricGroup().getIOMetricGroup()); } }
StreamInputProcessor
public void setMetricGroup(TaskIOMetricGroup metrics) { metrics.gauge("currentLowWatermark", new Gauge<Long>() { @Override public Long getValue() { return lastEmittedWatermark; } }); metrics.gauge("checkpointAlignmentTime", new Gauge<Long>() { @Override public Long getValue() { return barrierHandler.getAlignmentDurationNanos(); } }); }
currentLowWatermark，即lastEmittedWatermark
默认值是，
lastEmittedWatermark = Long.MIN_VALUE;
所以如果没有assignTimestampsAndWatermarks，那么currentLowWatermark会是一个极大的负数
public boolean processInput(OneInputStreamOperator<IN, ?> streamOperator, final Object lock) throws Exception { while (true) { if (currentRecordDeserializer != null) { if (result.isFullRecord()) { StreamElement recordOrMark = deserializationDelegate.getInstance(); if (recordOrMark.isWatermark()) { long watermarkMillis = recordOrMark.asWatermark().getTimestamp(); if (watermarkMillis > watermarks[currentChannel]) { // 更新每个channel对应的waterMark watermarks[currentChannel] = watermarkMillis; long newMinWatermark = Long.MAX_VALUE; for (long watermark: watermarks) { // 找出所有channel最小的watermark，以最小的为准 newMinWatermark = Math.min(watermark, newMinWatermark); } if (newMinWatermark > lastEmittedWatermark) { lastEmittedWatermark = newMinWatermark; // 将最小的watermark设为lastEmittedWatermark synchronized (lock) { streamOperator.processWatermark(new Watermark(lastEmittedWatermark)); } } } continue; }
checkpointAlignmentTime
barrierHandler.getAlignmentDurationNanos
@Override public long getAlignmentDurationNanos() { long start = this.startOfAlignmentTimestamp; if (start <= 0) { return latestAlignmentDurationNanos; } else { return System.nanoTime() - start; } }
startOfAlignmentTimestamp是在这次checkpoint开始的时候打的时间戳，即beginNewAlignment
private void beginNewAlignment(long checkpointId, int channelIndex) throws IOException { currentCheckpointId = checkpointId; onBarrier(channelIndex); startOfAlignmentTimestamp = System.nanoTime(); }
beginNewAlignment在
processBarrier中被调用，
if (numBarriersReceived > 0) { // this is only true if some alignment is already progress and was not canceled if (barrierId == currentCheckpointId) { // regular case onBarrier(channelIndex); } else if (barrierId > currentCheckpointId) {// 当收到新的checkpointid，所以老的id已经过期，需要产生新的checkpoint // we did not complete the current checkpoint, another started before LOG.warn("Received checkpoint barrier for checkpoint {} before completing current checkpoint {}. " + "Skipping current checkpoint.", barrierId, currentCheckpointId); // let the task know we are not completing this notifyAbort(currentCheckpointId, new CheckpointDeclineSubsumedException(barrierId)); // abort the current checkpoint releaseBlocksAndResetBarriers(); // begin a the new checkpoint beginNewAlignment(barrierId, channelIndex); //标识checkpoint开始 } else { // ignore trailing barrier from an earlier checkpoint (obsolete now) return; } } else if (barrierId > currentCheckpointId) { //新的checkpoint开始 // first barrier of a new checkpoint beginNewAlignment(barrierId, channelIndex); //标识checkpoint开始 }
所以checkpointAlignmentTime的意思是，当前的checkpoint已经等待多久，因为要等到所有input channel的barrier，checkpoint才会触发
单位是纳秒，所以billion级别代表秒
如果比较大，说明各个并发之前的延迟差异较大，或延迟较高