Java Code Examples for org.apache.flink.runtime.metrics.dump.QueryScopeInfo#OperatorQueryScopeInfo
The following examples show how to use
org.apache.flink.runtime.metrics.dump.QueryScopeInfo#OperatorQueryScopeInfo .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: OperatorGroupTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Test
public void testCreateQueryServiceMetricInfo() {
JobID jid = new JobID();
JobVertexID vid = new JobVertexID();
AbstractID eid = new AbstractID();
OperatorID oid = new OperatorID();
TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
OperatorMetricGroup operator = new OperatorMetricGroup(registry, task, oid, "operator");
QueryScopeInfo.OperatorQueryScopeInfo info = operator.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("", info.scope);
assertEquals(jid.toString(), info.jobID);
assertEquals(vid.toString(), info.vertexID);
assertEquals(4, info.subtaskIndex);
assertEquals("operator", info.operatorName);
}
Example 2
| Source File: OperatorGroupTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testCreateQueryServiceMetricInfo() {
JobID jid = new JobID();
JobVertexID vid = new JobVertexID();
AbstractID eid = new AbstractID();
OperatorID oid = new OperatorID();
TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
OperatorMetricGroup operator = new OperatorMetricGroup(registry, task, oid, "operator");
QueryScopeInfo.OperatorQueryScopeInfo info = operator.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("", info.scope);
assertEquals(jid.toString(), info.jobID);
assertEquals(vid.toString(), info.vertexID);
assertEquals(4, info.subtaskIndex);
assertEquals("operator", info.operatorName);
}
Example 3
| Source File: OperatorGroupTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testCreateQueryServiceMetricInfo() {
JobID jid = new JobID();
JobVertexID vid = new JobVertexID();
AbstractID eid = new AbstractID();
OperatorID oid = new OperatorID();
TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
OperatorMetricGroup operator = new OperatorMetricGroup(registry, task, oid, "operator");
QueryScopeInfo.OperatorQueryScopeInfo info = operator.createQueryServiceMetricInfo(new DummyCharacterFilter());
assertEquals("", info.scope);
assertEquals(jid.toString(), info.jobID);
assertEquals(vid.toString(), info.vertexID);
assertEquals(4, info.subtaskIndex);
assertEquals("operator", info.operatorName);
}
Example 4
| Source File: OperatorMetricGroup.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Override
protected QueryScopeInfo.OperatorQueryScopeInfo createQueryServiceMetricInfo(CharacterFilter filter) {
return new QueryScopeInfo.OperatorQueryScopeInfo(
this.parent.parent.jobId.toString(),
this.parent.vertexId.toString(),
this.parent.subtaskIndex,
filter.filterCharacters(this.operatorName));
}
Example 5
| Source File: OperatorMetricGroup.java From flink with Apache License 2.0 | 5 votes |
|
@Override
protected QueryScopeInfo.OperatorQueryScopeInfo createQueryServiceMetricInfo(CharacterFilter filter) {
return new QueryScopeInfo.OperatorQueryScopeInfo(
this.parent.parent.jobId.toString(),
this.parent.vertexId.toString(),
this.parent.subtaskIndex,
filter.filterCharacters(this.operatorName));
}
Example 6
| Source File: OperatorMetricGroup.java From flink with Apache License 2.0 | 5 votes |
|
@Override
protected QueryScopeInfo.OperatorQueryScopeInfo createQueryServiceMetricInfo(CharacterFilter filter) {
return new QueryScopeInfo.OperatorQueryScopeInfo(
this.parent.parent.jobId.toString(),
this.parent.vertexId.toString(),
this.parent.subtaskIndex,
filter.filterCharacters(this.operatorName));
}
Example 7
| Source File: MetricStore.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
public void add(MetricDump metric) {
try {
QueryScopeInfo info = metric.scopeInfo;
TaskManagerMetricStore tm;
JobMetricStore job;
TaskMetricStore task;
ComponentMetricStore subtask;
String name = info.scope.isEmpty()
? metric.name
: info.scope + "." + metric.name;
if (name.isEmpty()) { // malformed transmission
return;
}
switch (info.getCategory()) {
case INFO_CATEGORY_JM:
addMetric(jobManager.metrics, name, metric);
break;
case INFO_CATEGORY_TM:
String tmID = ((QueryScopeInfo.TaskManagerQueryScopeInfo) info).taskManagerID;
tm = taskManagers.computeIfAbsent(tmID, k -> new TaskManagerMetricStore());
if (name.contains("GarbageCollector")) {
String gcName = name.substring("Status.JVM.GarbageCollector.".length(), name.lastIndexOf('.'));
tm.addGarbageCollectorName(gcName);
}
addMetric(tm.metrics, name, metric);
break;
case INFO_CATEGORY_JOB:
QueryScopeInfo.JobQueryScopeInfo jobInfo = (QueryScopeInfo.JobQueryScopeInfo) info;
job = jobs.computeIfAbsent(jobInfo.jobID, k -> new JobMetricStore());
addMetric(job.metrics, name, metric);
break;
case INFO_CATEGORY_TASK:
QueryScopeInfo.TaskQueryScopeInfo taskInfo = (QueryScopeInfo.TaskQueryScopeInfo) info;
job = jobs.computeIfAbsent(taskInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(taskInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(taskInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* The duplication is intended. Metrics scoped by subtask are useful for several job/task handlers,
* while the WebInterface task metric queries currently do not account for subtasks, so we don't
* divide by subtask and instead use the concatenation of subtask index and metric name as the name
* for those.
*/
addMetric(subtask.metrics, name, metric);
addMetric(task.metrics, taskInfo.subtaskIndex + "." + name, metric);
break;
case INFO_CATEGORY_OPERATOR:
QueryScopeInfo.OperatorQueryScopeInfo operatorInfo = (QueryScopeInfo.OperatorQueryScopeInfo) info;
job = jobs.computeIfAbsent(operatorInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(operatorInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(operatorInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* As the WebInterface does not account for operators (because it can't) we don't
* divide by operator and instead use the concatenation of subtask index, operator name and metric name
* as the name.
*/
addMetric(subtask.metrics, operatorInfo.operatorName + "." + name, metric);
addMetric(task.metrics, operatorInfo.subtaskIndex + "." + operatorInfo.operatorName + "." + name, metric);
break;
default:
LOG.debug("Invalid metric dump category: " + info.getCategory());
}
} catch (Exception e) {
LOG.debug("Malformed metric dump.", e);
}
}
Example 8
| Source File: MetricStoreTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
public static MetricStore setupStore(MetricStore store) {
QueryScopeInfo.JobManagerQueryScopeInfo jm = new QueryScopeInfo.JobManagerQueryScopeInfo("abc");
MetricDump.CounterDump cd1 = new MetricDump.CounterDump(jm, "metric1", 0);
QueryScopeInfo.TaskManagerQueryScopeInfo tm = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid", "abc");
MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm, "metric2", 1);
MetricDump.CounterDump cd2a = new MetricDump.CounterDump(tm, "metric22", 1);
QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid2", "abc");
MetricDump.CounterDump cd22 = new MetricDump.CounterDump(tm2, "metric2", 10);
MetricDump.CounterDump cd22a = new MetricDump.CounterDump(tm2, "metric2b", 10);
QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo("jobid", "abc");
MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job, "metric3", 2);
MetricDump.CounterDump cd4 = new MetricDump.CounterDump(job, "metric4", 3);
QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo("jobid2", "abc");
MetricDump.CounterDump cd32 = new MetricDump.CounterDump(job2, "metric3", 2);
MetricDump.CounterDump cd42 = new MetricDump.CounterDump(job2, "metric4", 3);
QueryScopeInfo.TaskQueryScopeInfo task = new QueryScopeInfo.TaskQueryScopeInfo("jobid", "taskid", 8, "abc");
MetricDump.CounterDump cd5 = new MetricDump.CounterDump(task, "metric5", 4);
QueryScopeInfo.OperatorQueryScopeInfo operator = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 8, "opname", "abc");
MetricDump.CounterDump cd6 = new MetricDump.CounterDump(operator, "metric6", 5);
MetricDump.CounterDump cd7 = new MetricDump.CounterDump(operator, "metric7", 6);
QueryScopeInfo.OperatorQueryScopeInfo operator2 = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 1, "opname", "abc");
MetricDump.CounterDump cd62 = new MetricDump.CounterDump(operator2, "metric6", 5);
MetricDump.CounterDump cd72 = new MetricDump.CounterDump(operator2, "metric7", 6);
store.add(cd1);
store.add(cd2);
store.add(cd2a);
store.add(cd3);
store.add(cd4);
store.add(cd5);
store.add(cd6);
store.add(cd7);
store.add(cd62);
store.add(cd72);
store.add(cd22);
store.add(cd22a);
store.add(cd32);
store.add(cd42);
return store;
}
Example 9
| Source File: MetricStore.java From flink with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
public void add(MetricDump metric) {
try {
QueryScopeInfo info = metric.scopeInfo;
TaskManagerMetricStore tm;
JobMetricStore job;
TaskMetricStore task;
ComponentMetricStore subtask;
String name = info.scope.isEmpty()
? metric.name
: info.scope + "." + metric.name;
if (name.isEmpty()) { // malformed transmission
return;
}
switch (info.getCategory()) {
case INFO_CATEGORY_JM:
addMetric(jobManager.metrics, name, metric);
break;
case INFO_CATEGORY_TM:
String tmID = ((QueryScopeInfo.TaskManagerQueryScopeInfo) info).taskManagerID;
tm = taskManagers.computeIfAbsent(tmID, k -> new TaskManagerMetricStore());
if (name.contains("GarbageCollector")) {
String gcName = name.substring("Status.JVM.GarbageCollector.".length(), name.lastIndexOf('.'));
tm.addGarbageCollectorName(gcName);
}
addMetric(tm.metrics, name, metric);
break;
case INFO_CATEGORY_JOB:
QueryScopeInfo.JobQueryScopeInfo jobInfo = (QueryScopeInfo.JobQueryScopeInfo) info;
job = jobs.computeIfAbsent(jobInfo.jobID, k -> new JobMetricStore());
addMetric(job.metrics, name, metric);
break;
case INFO_CATEGORY_TASK:
QueryScopeInfo.TaskQueryScopeInfo taskInfo = (QueryScopeInfo.TaskQueryScopeInfo) info;
job = jobs.computeIfAbsent(taskInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(taskInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(taskInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* The duplication is intended. Metrics scoped by subtask are useful for several job/task handlers,
* while the WebInterface task metric queries currently do not account for subtasks, so we don't
* divide by subtask and instead use the concatenation of subtask index and metric name as the name
* for those.
*/
addMetric(subtask.metrics, name, metric);
addMetric(task.metrics, taskInfo.subtaskIndex + "." + name, metric);
break;
case INFO_CATEGORY_OPERATOR:
QueryScopeInfo.OperatorQueryScopeInfo operatorInfo = (QueryScopeInfo.OperatorQueryScopeInfo) info;
job = jobs.computeIfAbsent(operatorInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(operatorInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(operatorInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* As the WebInterface does not account for operators (because it can't) we don't
* divide by operator and instead use the concatenation of subtask index, operator name and metric name
* as the name.
*/
addMetric(subtask.metrics, operatorInfo.operatorName + "." + name, metric);
addMetric(task.metrics, operatorInfo.subtaskIndex + "." + operatorInfo.operatorName + "." + name, metric);
break;
default:
LOG.debug("Invalid metric dump category: " + info.getCategory());
}
} catch (Exception e) {
LOG.debug("Malformed metric dump.", e);
}
}
Example 10
| Source File: MetricStoreTest.java From flink with Apache License 2.0 | 4 votes |
|
public static MetricStore setupStore(MetricStore store) {
QueryScopeInfo.JobManagerQueryScopeInfo jm = new QueryScopeInfo.JobManagerQueryScopeInfo("abc");
MetricDump.CounterDump cd1 = new MetricDump.CounterDump(jm, "metric1", 0);
QueryScopeInfo.TaskManagerQueryScopeInfo tm = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid", "abc");
MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm, "metric2", 1);
MetricDump.CounterDump cd2a = new MetricDump.CounterDump(tm, "metric22", 1);
QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid2", "abc");
MetricDump.CounterDump cd22 = new MetricDump.CounterDump(tm2, "metric2", 10);
MetricDump.CounterDump cd22a = new MetricDump.CounterDump(tm2, "metric2b", 10);
QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo("jobid", "abc");
MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job, "metric3", 2);
MetricDump.CounterDump cd4 = new MetricDump.CounterDump(job, "metric4", 3);
QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo("jobid2", "abc");
MetricDump.CounterDump cd32 = new MetricDump.CounterDump(job2, "metric3", 2);
MetricDump.CounterDump cd42 = new MetricDump.CounterDump(job2, "metric4", 3);
QueryScopeInfo.TaskQueryScopeInfo task = new QueryScopeInfo.TaskQueryScopeInfo("jobid", "taskid", 8, "abc");
MetricDump.CounterDump cd5 = new MetricDump.CounterDump(task, "metric5", 4);
QueryScopeInfo.OperatorQueryScopeInfo operator = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 8, "opname", "abc");
MetricDump.CounterDump cd6 = new MetricDump.CounterDump(operator, "metric6", 5);
MetricDump.CounterDump cd7 = new MetricDump.CounterDump(operator, "metric7", 6);
QueryScopeInfo.OperatorQueryScopeInfo operator2 = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 1, "opname", "abc");
MetricDump.CounterDump cd62 = new MetricDump.CounterDump(operator2, "metric6", 5);
MetricDump.CounterDump cd72 = new MetricDump.CounterDump(operator2, "metric7", 6);
store.add(cd1);
store.add(cd2);
store.add(cd2a);
store.add(cd3);
store.add(cd4);
store.add(cd5);
store.add(cd6);
store.add(cd7);
store.add(cd62);
store.add(cd72);
store.add(cd22);
store.add(cd22a);
store.add(cd32);
store.add(cd42);
return store;
}
Example 11
| Source File: MetricStore.java From flink with Apache License 2.0 | 4 votes |
|
@VisibleForTesting
public void add(MetricDump metric) {
try {
QueryScopeInfo info = metric.scopeInfo;
TaskManagerMetricStore tm;
JobMetricStore job;
TaskMetricStore task;
ComponentMetricStore subtask;
String name = info.scope.isEmpty()
? metric.name
: info.scope + "." + metric.name;
if (name.isEmpty()) { // malformed transmission
return;
}
switch (info.getCategory()) {
case INFO_CATEGORY_JM:
addMetric(jobManager.metrics, name, metric);
break;
case INFO_CATEGORY_TM:
String tmID = ((QueryScopeInfo.TaskManagerQueryScopeInfo) info).taskManagerID;
tm = taskManagers.computeIfAbsent(tmID, k -> new TaskManagerMetricStore());
if (name.contains("GarbageCollector")) {
String gcName = name.substring("Status.JVM.GarbageCollector.".length(), name.lastIndexOf('.'));
tm.addGarbageCollectorName(gcName);
}
addMetric(tm.metrics, name, metric);
break;
case INFO_CATEGORY_JOB:
QueryScopeInfo.JobQueryScopeInfo jobInfo = (QueryScopeInfo.JobQueryScopeInfo) info;
job = jobs.computeIfAbsent(jobInfo.jobID, k -> new JobMetricStore());
addMetric(job.metrics, name, metric);
break;
case INFO_CATEGORY_TASK:
QueryScopeInfo.TaskQueryScopeInfo taskInfo = (QueryScopeInfo.TaskQueryScopeInfo) info;
job = jobs.computeIfAbsent(taskInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(taskInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(taskInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* The duplication is intended. Metrics scoped by subtask are useful for several job/task handlers,
* while the WebInterface task metric queries currently do not account for subtasks, so we don't
* divide by subtask and instead use the concatenation of subtask index and metric name as the name
* for those.
*/
addMetric(subtask.metrics, name, metric);
addMetric(task.metrics, taskInfo.subtaskIndex + "." + name, metric);
break;
case INFO_CATEGORY_OPERATOR:
QueryScopeInfo.OperatorQueryScopeInfo operatorInfo = (QueryScopeInfo.OperatorQueryScopeInfo) info;
job = jobs.computeIfAbsent(operatorInfo.jobID, k -> new JobMetricStore());
task = job.tasks.computeIfAbsent(operatorInfo.vertexID, k -> new TaskMetricStore());
subtask = task.subtasks.computeIfAbsent(operatorInfo.subtaskIndex, k -> new ComponentMetricStore());
/**
* As the WebInterface does not account for operators (because it can't) we don't
* divide by operator and instead use the concatenation of subtask index, operator name and metric name
* as the name.
*/
addMetric(subtask.metrics, operatorInfo.operatorName + "." + name, metric);
addMetric(task.metrics, operatorInfo.subtaskIndex + "." + operatorInfo.operatorName + "." + name, metric);
break;
default:
LOG.debug("Invalid metric dump category: " + info.getCategory());
}
} catch (Exception e) {
LOG.debug("Malformed metric dump.", e);
}
}
Example 12
| Source File: MetricStoreTest.java From flink with Apache License 2.0 | 4 votes |
|
public static MetricStore setupStore(MetricStore store) {
QueryScopeInfo.JobManagerQueryScopeInfo jm = new QueryScopeInfo.JobManagerQueryScopeInfo("abc");
MetricDump.CounterDump cd1 = new MetricDump.CounterDump(jm, "metric1", 0);
QueryScopeInfo.TaskManagerQueryScopeInfo tm = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid", "abc");
MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm, "metric2", 1);
MetricDump.CounterDump cd2a = new MetricDump.CounterDump(tm, "metric22", 1);
QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid2", "abc");
MetricDump.CounterDump cd22 = new MetricDump.CounterDump(tm2, "metric2", 10);
MetricDump.CounterDump cd22a = new MetricDump.CounterDump(tm2, "metric2b", 10);
QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo("jobid", "abc");
MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job, "metric3", 2);
MetricDump.CounterDump cd4 = new MetricDump.CounterDump(job, "metric4", 3);
QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo("jobid2", "abc");
MetricDump.CounterDump cd32 = new MetricDump.CounterDump(job2, "metric3", 2);
MetricDump.CounterDump cd42 = new MetricDump.CounterDump(job2, "metric4", 3);
QueryScopeInfo.TaskQueryScopeInfo task = new QueryScopeInfo.TaskQueryScopeInfo("jobid", "taskid", 8, "abc");
MetricDump.CounterDump cd5 = new MetricDump.CounterDump(task, "metric5", 4);
QueryScopeInfo.OperatorQueryScopeInfo operator = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 8, "opname", "abc");
MetricDump.CounterDump cd6 = new MetricDump.CounterDump(operator, "metric6", 5);
MetricDump.CounterDump cd7 = new MetricDump.CounterDump(operator, "metric7", 6);
QueryScopeInfo.OperatorQueryScopeInfo operator2 = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 1, "opname", "abc");
MetricDump.CounterDump cd62 = new MetricDump.CounterDump(operator2, "metric6", 5);
MetricDump.CounterDump cd72 = new MetricDump.CounterDump(operator2, "metric7", 6);
store.add(cd1);
store.add(cd2);
store.add(cd2a);
store.add(cd3);
store.add(cd4);
store.add(cd5);
store.add(cd6);
store.add(cd7);
store.add(cd62);
store.add(cd72);
store.add(cd22);
store.add(cd22a);
store.add(cd32);
store.add(cd42);
return store;
}
