Java Code Examples for org.apache.flink.runtime.metrics.dump.QueryScopeInfo#TaskQueryScopeInfo
The following examples show how to use
org.apache.flink.runtime.metrics.dump.QueryScopeInfo#TaskQueryScopeInfo .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: MetricGroupTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Test
public void testCreateQueryServiceMetricInfo() {
JobID jid = new JobID();
JobVertexID vid = new JobVertexID();
AbstractID eid = new AbstractID();
MetricRegistryImpl registry = new MetricRegistryImpl(defaultMetricRegistryConfiguration);
TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
GenericMetricGroup userGroup1 = new GenericMetricGroup(registry, task, "hello");
GenericMetricGroup userGroup2 = new GenericMetricGroup(registry, userGroup1, "world");
QueryScopeInfo.TaskQueryScopeInfo info1 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup1.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("hello", info1.scope);
assertEquals(jid.toString(), info1.jobID);
assertEquals(vid.toString(), info1.vertexID);
assertEquals(4, info1.subtaskIndex);
QueryScopeInfo.TaskQueryScopeInfo info2 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup2.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("hello.world", info2.scope);
assertEquals(jid.toString(), info2.jobID);
assertEquals(vid.toString(), info2.vertexID);
assertEquals(4, info2.subtaskIndex);
}
Example 2
| Source File: MetricGroupTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testCreateQueryServiceMetricInfo() {
JobID jid = new JobID();
JobVertexID vid = new JobVertexID();
AbstractID eid = new AbstractID();
MetricRegistryImpl registry = new MetricRegistryImpl(defaultMetricRegistryConfiguration);
TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
GenericMetricGroup userGroup1 = new GenericMetricGroup(registry, task, "hello");
GenericMetricGroup userGroup2 = new GenericMetricGroup(registry, userGroup1, "world");
QueryScopeInfo.TaskQueryScopeInfo info1 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup1.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("hello", info1.scope);
assertEquals(jid.toString(), info1.jobID);
assertEquals(vid.toString(), info1.vertexID);
assertEquals(4, info1.subtaskIndex);
QueryScopeInfo.TaskQueryScopeInfo info2 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup2.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("hello.world", info2.scope);
assertEquals(jid.toString(), info2.jobID);
assertEquals(vid.toString(), info2.vertexID);
assertEquals(4, info2.subtaskIndex);
}
Example 3
| Source File: AggregatingSubtasksMetricsHandlerTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Override
protected Collection<MetricDump> getMetricDumps() {
Collection<MetricDump> dumps = new ArrayList<>(3);
QueryScopeInfo.TaskQueryScopeInfo task1 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 1, "abc");
MetricDump.CounterDump cd1 = new MetricDump.CounterDump(task1, "metric1", 1);
dumps.add(cd1);
QueryScopeInfo.TaskQueryScopeInfo task2 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 2, "abc");
MetricDump.CounterDump cd2 = new MetricDump.CounterDump(task2, "metric1", 3);
dumps.add(cd2);
QueryScopeInfo.TaskQueryScopeInfo task3 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 3, "abc");
MetricDump.CounterDump cd3 = new MetricDump.CounterDump(task3, "metric2", 5);
dumps.add(cd3);
return dumps;
}
Example 4
| Source File: AggregatingSubtasksMetricsHandlerTest.java From flink with Apache License 2.0 | 6 votes |
|
@Override
protected Collection<MetricDump> getMetricDumps() {
Collection<MetricDump> dumps = new ArrayList<>(3);
QueryScopeInfo.TaskQueryScopeInfo task1 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 1, "abc");
MetricDump.CounterDump cd1 = new MetricDump.CounterDump(task1, "metric1", 1);
dumps.add(cd1);
QueryScopeInfo.TaskQueryScopeInfo task2 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 2, "abc");
MetricDump.CounterDump cd2 = new MetricDump.CounterDump(task2, "metric1", 3);
dumps.add(cd2);
QueryScopeInfo.TaskQueryScopeInfo task3 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 3, "abc");
MetricDump.CounterDump cd3 = new MetricDump.CounterDump(task3, "metric2", 5);
dumps.add(cd3);
return dumps;
}
Example 5
| Source File: MetricGroupTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testCreateQueryServiceMetricInfo() {
JobID jid = new JobID();
JobVertexID vid = new JobVertexID();
AbstractID eid = new AbstractID();
MetricRegistryImpl registry = new MetricRegistryImpl(defaultMetricRegistryConfiguration);
TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
GenericMetricGroup userGroup1 = new GenericMetricGroup(registry, task, "hello");
GenericMetricGroup userGroup2 = new GenericMetricGroup(registry, userGroup1, "world");
QueryScopeInfo.TaskQueryScopeInfo info1 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup1.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("hello", info1.scope);
assertEquals(jid.toString(), info1.jobID);
assertEquals(vid.toString(), info1.vertexID);
assertEquals(4, info1.subtaskIndex);
QueryScopeInfo.TaskQueryScopeInfo info2 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup2.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("hello.world", info2.scope);
assertEquals(jid.toString(), info2.jobID);
assertEquals(vid.toString(), info2.vertexID);
assertEquals(4, info2.subtaskIndex);
}
Example 6
| Source File: TaskMetricGroupTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testCreateQueryServiceMetricInfo() {
JobID jid = new JobID();
JobVertexID vid = new JobVertexID();
AbstractID eid = new AbstractID();
TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
QueryScopeInfo.TaskQueryScopeInfo info = task.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("", info.scope);
assertEquals(jid.toString(), info.jobID);
assertEquals(vid.toString(), info.vertexID);
assertEquals(4, info.subtaskIndex);
}
Example 7
| Source File: TaskMetricGroup.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Override
protected QueryScopeInfo.TaskQueryScopeInfo createQueryServiceMetricInfo(CharacterFilter filter) {
return new QueryScopeInfo.TaskQueryScopeInfo(
this.parent.jobId.toString(),
String.valueOf(this.vertexId),
this.subtaskIndex);
}
Example 8
| Source File: TaskMetricGroup.java From flink with Apache License 2.0 | 5 votes |
|
@Override
protected QueryScopeInfo.TaskQueryScopeInfo createQueryServiceMetricInfo(CharacterFilter filter) {
return new QueryScopeInfo.TaskQueryScopeInfo(
this.parent.jobId.toString(),
String.valueOf(this.vertexId),
this.subtaskIndex);
}
Example 9
| Source File: TaskMetricGroupTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testCreateQueryServiceMetricInfo() {
JobID jid = new JobID();
JobVertexID vid = new JobVertexID();
AbstractID eid = new AbstractID();
TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
QueryScopeInfo.TaskQueryScopeInfo info = task.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("", info.scope);
assertEquals(jid.toString(), info.jobID);
assertEquals(vid.toString(), info.vertexID);
assertEquals(4, info.subtaskIndex);
}
Example 10
| Source File: MetricStore.java From flink with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
public void add(MetricDump metric) {
try {
QueryScopeInfo info = metric.scopeInfo;
TaskManagerMetricStore tm;
JobMetricStore job;
TaskMetricStore task;
ComponentMetricStore subtask;
String name = info.scope.isEmpty()
? metric.name
: info.scope + "." + metric.name;
if (name.isEmpty()) { // malformed transmission
return;
}
switch (info.getCategory()) {
case INFO_CATEGORY_JM:
addMetric(jobManager.metrics, name, metric);
break;
case INFO_CATEGORY_TM:
String tmID = ((QueryScopeInfo.TaskManagerQueryScopeInfo) info).taskManagerID;
tm = taskManagers.computeIfAbsent(tmID, k -> new TaskManagerMetricStore());
if (name.contains("GarbageCollector")) {
String gcName = name.substring("Status.JVM.GarbageCollector.".length(), name.lastIndexOf('.'));
tm.addGarbageCollectorName(gcName);
}
addMetric(tm.metrics, name, metric);
break;
case INFO_CATEGORY_JOB:
QueryScopeInfo.JobQueryScopeInfo jobInfo = (QueryScopeInfo.JobQueryScopeInfo) info;
job = jobs.computeIfAbsent(jobInfo.jobID, k -> new JobMetricStore());
addMetric(job.metrics, name, metric);
break;
case INFO_CATEGORY_TASK:
QueryScopeInfo.TaskQueryScopeInfo taskInfo = (QueryScopeInfo.TaskQueryScopeInfo) info;
job = jobs.computeIfAbsent(taskInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(taskInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(taskInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* The duplication is intended. Metrics scoped by subtask are useful for several job/task handlers,
* while the WebInterface task metric queries currently do not account for subtasks, so we don't
* divide by subtask and instead use the concatenation of subtask index and metric name as the name
* for those.
*/
addMetric(subtask.metrics, name, metric);
addMetric(task.metrics, taskInfo.subtaskIndex + "." + name, metric);
break;
case INFO_CATEGORY_OPERATOR:
QueryScopeInfo.OperatorQueryScopeInfo operatorInfo = (QueryScopeInfo.OperatorQueryScopeInfo) info;
job = jobs.computeIfAbsent(operatorInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(operatorInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(operatorInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* As the WebInterface does not account for operators (because it can't) we don't
* divide by operator and instead use the concatenation of subtask index, operator name and metric name
* as the name.
*/
addMetric(subtask.metrics, operatorInfo.operatorName + "." + name, metric);
addMetric(task.metrics, operatorInfo.subtaskIndex + "." + operatorInfo.operatorName + "." + name, metric);
break;
default:
LOG.debug("Invalid metric dump category: " + info.getCategory());
}
} catch (Exception e) {
LOG.debug("Malformed metric dump.", e);
}
}
Example 11
| Source File: JobVertexMetricsHandlerTest.java From flink with Apache License 2.0 | 4 votes |
|
@Override
QueryScopeInfo getQueryScopeInfo() {
return new QueryScopeInfo.TaskQueryScopeInfo(TEST_JOB_ID, TEST_VERTEX_ID, TEST_SUBTASK_INDEX);
}
Example 12
| Source File: MetricStoreTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
public static MetricStore setupStore(MetricStore store) {
QueryScopeInfo.JobManagerQueryScopeInfo jm = new QueryScopeInfo.JobManagerQueryScopeInfo("abc");
MetricDump.CounterDump cd1 = new MetricDump.CounterDump(jm, "metric1", 0);
QueryScopeInfo.TaskManagerQueryScopeInfo tm = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid", "abc");
MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm, "metric2", 1);
MetricDump.CounterDump cd2a = new MetricDump.CounterDump(tm, "metric22", 1);
QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid2", "abc");
MetricDump.CounterDump cd22 = new MetricDump.CounterDump(tm2, "metric2", 10);
MetricDump.CounterDump cd22a = new MetricDump.CounterDump(tm2, "metric2b", 10);
QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo("jobid", "abc");
MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job, "metric3", 2);
MetricDump.CounterDump cd4 = new MetricDump.CounterDump(job, "metric4", 3);
QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo("jobid2", "abc");
MetricDump.CounterDump cd32 = new MetricDump.CounterDump(job2, "metric3", 2);
MetricDump.CounterDump cd42 = new MetricDump.CounterDump(job2, "metric4", 3);
QueryScopeInfo.TaskQueryScopeInfo task = new QueryScopeInfo.TaskQueryScopeInfo("jobid", "taskid", 8, "abc");
MetricDump.CounterDump cd5 = new MetricDump.CounterDump(task, "metric5", 4);
QueryScopeInfo.OperatorQueryScopeInfo operator = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 8, "opname", "abc");
MetricDump.CounterDump cd6 = new MetricDump.CounterDump(operator, "metric6", 5);
MetricDump.CounterDump cd7 = new MetricDump.CounterDump(operator, "metric7", 6);
QueryScopeInfo.OperatorQueryScopeInfo operator2 = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 1, "opname", "abc");
MetricDump.CounterDump cd62 = new MetricDump.CounterDump(operator2, "metric6", 5);
MetricDump.CounterDump cd72 = new MetricDump.CounterDump(operator2, "metric7", 6);
store.add(cd1);
store.add(cd2);
store.add(cd2a);
store.add(cd3);
store.add(cd4);
store.add(cd5);
store.add(cd6);
store.add(cd7);
store.add(cd62);
store.add(cd72);
store.add(cd22);
store.add(cd22a);
store.add(cd32);
store.add(cd42);
return store;
}
Example 13
| Source File: JobVertexMetricsHandlerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Override
QueryScopeInfo getQueryScopeInfo() {
return new QueryScopeInfo.TaskQueryScopeInfo(TEST_JOB_ID, TEST_VERTEX_ID, TEST_SUBTASK_INDEX);
}
Example 14
| Source File: SubtaskMetricsHandlerTest.java From flink with Apache License 2.0 | 4 votes |
|
@Override
QueryScopeInfo getQueryScopeInfo() {
return new QueryScopeInfo.TaskQueryScopeInfo(TEST_JOB_ID, TEST_VERTEX_ID,
TEST_SUBTASK_INDEX);
}
Example 15
| Source File: JobVertexMetricsHandlerTest.java From flink with Apache License 2.0 | 4 votes |
|
@Override
QueryScopeInfo getQueryScopeInfo() {
return new QueryScopeInfo.TaskQueryScopeInfo(TEST_JOB_ID, TEST_VERTEX_ID, TEST_SUBTASK_INDEX);
}
Example 16
| Source File: MetricStoreTest.java From flink with Apache License 2.0 | 4 votes |
|
public static MetricStore setupStore(MetricStore store) {
QueryScopeInfo.JobManagerQueryScopeInfo jm = new QueryScopeInfo.JobManagerQueryScopeInfo("abc");
MetricDump.CounterDump cd1 = new MetricDump.CounterDump(jm, "metric1", 0);
QueryScopeInfo.TaskManagerQueryScopeInfo tm = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid", "abc");
MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm, "metric2", 1);
MetricDump.CounterDump cd2a = new MetricDump.CounterDump(tm, "metric22", 1);
QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid2", "abc");
MetricDump.CounterDump cd22 = new MetricDump.CounterDump(tm2, "metric2", 10);
MetricDump.CounterDump cd22a = new MetricDump.CounterDump(tm2, "metric2b", 10);
QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo("jobid", "abc");
MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job, "metric3", 2);
MetricDump.CounterDump cd4 = new MetricDump.CounterDump(job, "metric4", 3);
QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo("jobid2", "abc");
MetricDump.CounterDump cd32 = new MetricDump.CounterDump(job2, "metric3", 2);
MetricDump.CounterDump cd42 = new MetricDump.CounterDump(job2, "metric4", 3);
QueryScopeInfo.TaskQueryScopeInfo task = new QueryScopeInfo.TaskQueryScopeInfo("jobid", "taskid", 8, "abc");
MetricDump.CounterDump cd5 = new MetricDump.CounterDump(task, "metric5", 4);
QueryScopeInfo.OperatorQueryScopeInfo operator = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 8, "opname", "abc");
MetricDump.CounterDump cd6 = new MetricDump.CounterDump(operator, "metric6", 5);
MetricDump.CounterDump cd7 = new MetricDump.CounterDump(operator, "metric7", 6);
QueryScopeInfo.OperatorQueryScopeInfo operator2 = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 1, "opname", "abc");
MetricDump.CounterDump cd62 = new MetricDump.CounterDump(operator2, "metric6", 5);
MetricDump.CounterDump cd72 = new MetricDump.CounterDump(operator2, "metric7", 6);
store.add(cd1);
store.add(cd2);
store.add(cd2a);
store.add(cd3);
store.add(cd4);
store.add(cd5);
store.add(cd6);
store.add(cd7);
store.add(cd62);
store.add(cd72);
store.add(cd22);
store.add(cd22a);
store.add(cd32);
store.add(cd42);
return store;
}
Example 17
| Source File: SubtaskMetricsHandlerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Override
QueryScopeInfo getQueryScopeInfo() {
return new QueryScopeInfo.TaskQueryScopeInfo(TEST_JOB_ID, TEST_VERTEX_ID,
TEST_SUBTASK_INDEX);
}
Example 18
| Source File: MetricStore.java From flink with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
public void add(MetricDump metric) {
try {
QueryScopeInfo info = metric.scopeInfo;
TaskManagerMetricStore tm;
JobMetricStore job;
TaskMetricStore task;
ComponentMetricStore subtask;
String name = info.scope.isEmpty()
? metric.name
: info.scope + "." + metric.name;
if (name.isEmpty()) { // malformed transmission
return;
}
switch (info.getCategory()) {
case INFO_CATEGORY_JM:
addMetric(jobManager.metrics, name, metric);
break;
case INFO_CATEGORY_TM:
String tmID = ((QueryScopeInfo.TaskManagerQueryScopeInfo) info).taskManagerID;
tm = taskManagers.computeIfAbsent(tmID, k -> new TaskManagerMetricStore());
if (name.contains("GarbageCollector")) {
String gcName = name.substring("Status.JVM.GarbageCollector.".length(), name.lastIndexOf('.'));
tm.addGarbageCollectorName(gcName);
}
addMetric(tm.metrics, name, metric);
break;
case INFO_CATEGORY_JOB:
QueryScopeInfo.JobQueryScopeInfo jobInfo = (QueryScopeInfo.JobQueryScopeInfo) info;
job = jobs.computeIfAbsent(jobInfo.jobID, k -> new JobMetricStore());
addMetric(job.metrics, name, metric);
break;
case INFO_CATEGORY_TASK:
QueryScopeInfo.TaskQueryScopeInfo taskInfo = (QueryScopeInfo.TaskQueryScopeInfo) info;
job = jobs.computeIfAbsent(taskInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(taskInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(taskInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* The duplication is intended. Metrics scoped by subtask are useful for several job/task handlers,
* while the WebInterface task metric queries currently do not account for subtasks, so we don't
* divide by subtask and instead use the concatenation of subtask index and metric name as the name
* for those.
*/
addMetric(subtask.metrics, name, metric);
addMetric(task.metrics, taskInfo.subtaskIndex + "." + name, metric);
break;
case INFO_CATEGORY_OPERATOR:
QueryScopeInfo.OperatorQueryScopeInfo operatorInfo = (QueryScopeInfo.OperatorQueryScopeInfo) info;
job = jobs.computeIfAbsent(operatorInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(operatorInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(operatorInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* As the WebInterface does not account for operators (because it can't) we don't
* divide by operator and instead use the concatenation of subtask index, operator name and metric name
* as the name.
*/
addMetric(subtask.metrics, operatorInfo.operatorName + "." + name, metric);
addMetric(task.metrics, operatorInfo.subtaskIndex + "." + operatorInfo.operatorName + "." + name, metric);
break;
default:
LOG.debug("Invalid metric dump category: " + info.getCategory());
}
} catch (Exception e) {
LOG.debug("Malformed metric dump.", e);
}
}
Example 19
| Source File: MetricStoreTest.java From flink with Apache License 2.0 | 4 votes |
|
public static MetricStore setupStore(MetricStore store) {
QueryScopeInfo.JobManagerQueryScopeInfo jm = new QueryScopeInfo.JobManagerQueryScopeInfo("abc");
MetricDump.CounterDump cd1 = new MetricDump.CounterDump(jm, "metric1", 0);
QueryScopeInfo.TaskManagerQueryScopeInfo tm = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid", "abc");
MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm, "metric2", 1);
MetricDump.CounterDump cd2a = new MetricDump.CounterDump(tm, "metric22", 1);
QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid2", "abc");
MetricDump.CounterDump cd22 = new MetricDump.CounterDump(tm2, "metric2", 10);
MetricDump.CounterDump cd22a = new MetricDump.CounterDump(tm2, "metric2b", 10);
QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo("jobid", "abc");
MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job, "metric3", 2);
MetricDump.CounterDump cd4 = new MetricDump.CounterDump(job, "metric4", 3);
QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo("jobid2", "abc");
MetricDump.CounterDump cd32 = new MetricDump.CounterDump(job2, "metric3", 2);
MetricDump.CounterDump cd42 = new MetricDump.CounterDump(job2, "metric4", 3);
QueryScopeInfo.TaskQueryScopeInfo task = new QueryScopeInfo.TaskQueryScopeInfo("jobid", "taskid", 8, "abc");
MetricDump.CounterDump cd5 = new MetricDump.CounterDump(task, "metric5", 4);
QueryScopeInfo.OperatorQueryScopeInfo operator = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 8, "opname", "abc");
MetricDump.CounterDump cd6 = new MetricDump.CounterDump(operator, "metric6", 5);
MetricDump.CounterDump cd7 = new MetricDump.CounterDump(operator, "metric7", 6);
QueryScopeInfo.OperatorQueryScopeInfo operator2 = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 1, "opname", "abc");
MetricDump.CounterDump cd62 = new MetricDump.CounterDump(operator2, "metric6", 5);
MetricDump.CounterDump cd72 = new MetricDump.CounterDump(operator2, "metric7", 6);
store.add(cd1);
store.add(cd2);
store.add(cd2a);
store.add(cd3);
store.add(cd4);
store.add(cd5);
store.add(cd6);
store.add(cd7);
store.add(cd62);
store.add(cd72);
store.add(cd22);
store.add(cd22a);
store.add(cd32);
store.add(cd42);
return store;
}
Example 20
| Source File: MetricStore.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
public void add(MetricDump metric) {
try {
QueryScopeInfo info = metric.scopeInfo;
TaskManagerMetricStore tm;
JobMetricStore job;
TaskMetricStore task;
ComponentMetricStore subtask;
String name = info.scope.isEmpty()
? metric.name
: info.scope + "." + metric.name;
if (name.isEmpty()) { // malformed transmission
return;
}
switch (info.getCategory()) {
case INFO_CATEGORY_JM:
addMetric(jobManager.metrics, name, metric);
break;
case INFO_CATEGORY_TM:
String tmID = ((QueryScopeInfo.TaskManagerQueryScopeInfo) info).taskManagerID;
tm = taskManagers.computeIfAbsent(tmID, k -> new TaskManagerMetricStore());
if (name.contains("GarbageCollector")) {
String gcName = name.substring("Status.JVM.GarbageCollector.".length(), name.lastIndexOf('.'));
tm.addGarbageCollectorName(gcName);
}
addMetric(tm.metrics, name, metric);
break;
case INFO_CATEGORY_JOB:
QueryScopeInfo.JobQueryScopeInfo jobInfo = (QueryScopeInfo.JobQueryScopeInfo) info;
job = jobs.computeIfAbsent(jobInfo.jobID, k -> new JobMetricStore());
addMetric(job.metrics, name, metric);
break;
case INFO_CATEGORY_TASK:
QueryScopeInfo.TaskQueryScopeInfo taskInfo = (QueryScopeInfo.TaskQueryScopeInfo) info;
job = jobs.computeIfAbsent(taskInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(taskInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(taskInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* The duplication is intended. Metrics scoped by subtask are useful for several job/task handlers,
* while the WebInterface task metric queries currently do not account for subtasks, so we don't
* divide by subtask and instead use the concatenation of subtask index and metric name as the name
* for those.
*/
addMetric(subtask.metrics, name, metric);
addMetric(task.metrics, taskInfo.subtaskIndex + "." + name, metric);
break;
case INFO_CATEGORY_OPERATOR:
QueryScopeInfo.OperatorQueryScopeInfo operatorInfo = (QueryScopeInfo.OperatorQueryScopeInfo) info;
job = jobs.computeIfAbsent(operatorInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(operatorInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(operatorInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* As the WebInterface does not account for operators (because it can't) we don't
* divide by operator and instead use the concatenation of subtask index, operator name and metric name
* as the name.
*/
addMetric(subtask.metrics, operatorInfo.operatorName + "." + name, metric);
addMetric(task.metrics, operatorInfo.subtaskIndex + "." + operatorInfo.operatorName + "." + name, metric);
break;
default:
LOG.debug("Invalid metric dump category: " + info.getCategory());
}
} catch (Exception e) {
LOG.debug("Malformed metric dump.", e);
}
}
