org.apache.flink.api.common.accumulators.LongCounter Java Examples

public static void main(String[] args) throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSource<String> data = env.fromElements("hadoop", "spark", "flink", "strom", "pyspark");

    data.map(new RichMapFunction<String, String>() {
        LongCounter counter = new LongCounter();

        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            getRuntimeContext().addAccumulator("ele_counter_java", counter);
        }

        @Override
        public String map(String value) throws Exception {
            counter.add(1);
            return value;
        }
    }).writeAsText("file:\\D:\\imooc\\新一代大数据计算引擎 Flink从入门到实战-v\\input\\sinkout\\sink-java-counter.txt",
            FileSystem.WriteMode.OVERWRITE).setParallelism(3);

    JobExecutionResult counterApp = env.execute("JavaCounterApp");
    Long num = counterApp.getAccumulatorResult("ele_counter_java");
    System.out.println("num:" + num);
}
 

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator("vc", new LongCounter(vertexCount));
	addAccumulator("uec", new LongCounter(unidirectionalEdgeCount));
	addAccumulator("bec", new LongCounter(bidirectionalEdgeCount));
	addAccumulator("021d", new LongCounter(triplet021dCount));
	addAccumulator("021u", new LongCounter(triplet021uCount));
	addAccumulator("021c", new LongCounter(triplet021cCount));
	addAccumulator("111d", new LongCounter(triplet111dCount));
	addAccumulator("111u", new LongCounter(triplet111uCount));
	addAccumulator("201", new LongCounter(triplet201Count));
}
 

@Test
public void testAccumulatorsAfterNoOp() {

	final String accName = "test_accumulator";

	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(6);

		env.generateSequence(1, 1000000)
				.rebalance()
				.flatMap(new RichFlatMapFunction<Long, Long>() {

					private LongCounter counter;

					@Override
					public void open(Configuration parameters) {
						counter = getRuntimeContext().getLongCounter(accName);
					}

					@Override
					public void flatMap(Long value, Collector<Long> out) {
						counter.add(1L);
					}
				})
				.output(new DiscardingOutputFormat<Long>());

		JobExecutionResult result = env.execute();

		assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public FlatOutputMapPartitionFunction(SerializableConfiguration conf, String cubeName, String segmentId,
                                      String metaUrl, int samplingPercentage, String bytesWrittenName,
                                      String recordCounterName) {
    this.cubeName = cubeName;
    this.segmentId = segmentId;
    this.metaUrl = metaUrl;
    this.conf = conf;
    this.samplingPercentage = samplingPercentage;
    this.bytesWrittenName = bytesWrittenName;
    this.recordCounterName = recordCounterName;
    this.bytesWrittenCounter = new LongCounter();
    this.recordCounter = new LongCounter();
}
 

@Test
public void testAccumulatorsAfterNoOp() {

	final String accName = "test_accumulator";

	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(6);
		env.getConfig().disableSysoutLogging();

		env.generateSequence(1, 1000000)
				.rebalance()
				.flatMap(new RichFlatMapFunction<Long, Long>() {

					private LongCounter counter;

					@Override
					public void open(Configuration parameters) {
						counter = getRuntimeContext().getLongCounter(accName);
					}

					@Override
					public void flatMap(Long value, Collector<Long> out) {
						counter.add(1L);
					}
				})
				.output(new DiscardingOutputFormat<Long>());

		JobExecutionResult result = env.execute();

		assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(UNIDIRECTIONAL_EDGE_COUNT, new LongCounter(unidirectionalEdgeCount));
	addAccumulator(BIDIRECTIONAL_EDGE_COUNT, new LongCounter(bidirectionalEdgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_OUT_DEGREE, new LongMaximum(maximumOutDegree));
	addAccumulator(MAXIMUM_IN_DEGREE, new LongMaximum(maximumInDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(EDGE_COUNT, new LongCounter(edgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator("vc", new LongCounter(vertexCount));
	addAccumulator("uec", new LongCounter(unidirectionalEdgeCount));
	addAccumulator("bec", new LongCounter(bidirectionalEdgeCount));
	addAccumulator("021d", new LongCounter(triplet021dCount));
	addAccumulator("021u", new LongCounter(triplet021uCount));
	addAccumulator("021c", new LongCounter(triplet021cCount));
	addAccumulator("111d", new LongCounter(triplet111dCount));
	addAccumulator("111u", new LongCounter(triplet111uCount));
	addAccumulator("201", new LongCounter(triplet201Count));
}
 

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(EDGE_COUNT, new LongCounter(edgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(UNIDIRECTIONAL_EDGE_COUNT, new LongCounter(unidirectionalEdgeCount));
	addAccumulator(BIDIRECTIONAL_EDGE_COUNT, new LongCounter(bidirectionalEdgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_OUT_DEGREE, new LongMaximum(maximumOutDegree));
	addAccumulator(MAXIMUM_IN_DEGREE, new LongMaximum(maximumInDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(UNIDIRECTIONAL_EDGE_COUNT, new LongCounter(unidirectionalEdgeCount));
	addAccumulator(BIDIRECTIONAL_EDGE_COUNT, new LongCounter(bidirectionalEdgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_OUT_DEGREE, new LongMaximum(maximumOutDegree));
	addAccumulator(MAXIMUM_IN_DEGREE, new LongMaximum(maximumInDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator("vc", new LongCounter(vertexCount));
	addAccumulator("uec", new LongCounter(unidirectionalEdgeCount));
	addAccumulator("bec", new LongCounter(bidirectionalEdgeCount));
	addAccumulator("021d", new LongCounter(triplet021dCount));
	addAccumulator("021u", new LongCounter(triplet021uCount));
	addAccumulator("021c", new LongCounter(triplet021cCount));
	addAccumulator("111d", new LongCounter(triplet111dCount));
	addAccumulator("111u", new LongCounter(triplet111uCount));
	addAccumulator("201", new LongCounter(triplet201Count));
}
 

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator(VERTEX_COUNT, new LongCounter(vertexCount));
	addAccumulator(EDGE_COUNT, new LongCounter(edgeCount));
	addAccumulator(TRIPLET_COUNT, new LongCounter(tripletCount));
	addAccumulator(MAXIMUM_DEGREE, new LongMaximum(maximumDegree));
	addAccumulator(MAXIMUM_TRIPLETS, new LongMaximum(maximumTriplets));
}
 

@Override
public void close() throws IOException {
	addAccumulator(TRIANGLE_TRIPLET_COUNT, new LongCounter(triangleTripletCount));
	addAccumulator(RECTANGLE_TRIPLET_COUNT, new LongCounter(rectangleTripletCount));
	addAccumulator(MAXIMUM_TRIANGLE_TRIPLETS, new LongMaximum(maximumTriangleTriplets));
	addAccumulator(MAXIMUM_RECTANGLE_TRIPLETS, new LongMaximum(maximumRectangleTriplets));
}
 

@Override
public void increment(String group, String counter, long l) {
	if(this.runtimeContext != null) {
		LongCounter flinkCounter = getOrInitCounter(EnumStringConverter.mergeGroupCounter(group, counter));
		flinkCounter.add(l);
	}
}
 

public static void increment(RuntimeContext runtimeContext, String group, String counter,
        long l) {
    if (runtimeContext != null) {
        LongCounter flinkCounter = getOrInitCounter(runtimeContext,
                mergeGroupCounter(group, counter));
        flinkCounter.add(l);
    }
}
 

@Test
public void testAccumulatorsAfterNoOp() {

	final String accName = "test_accumulator";

	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(6);
		env.getConfig().disableSysoutLogging();

		env.generateSequence(1, 1000000)
				.rebalance()
				.flatMap(new RichFlatMapFunction<Long, Long>() {

					private LongCounter counter;

					@Override
					public void open(Configuration parameters) {
						counter = getRuntimeContext().getLongCounter(accName);
					}

					@Override
					public void flatMap(Long value, Collector<Long> out) {
						counter.add(1L);
					}
				})
				.output(new DiscardingOutputFormat<Long>());

		JobExecutionResult result = env.execute();

		assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Override
public LongCounter clone() {
	throw new CustomException();
}
 

@Override
public void close() throws IOException {
	addAccumulator(COUNT, new LongCounter(count));
}
 

@Override
public LongCounter getLongCounter(String name) {
  throw new UnsupportedOperationException();
}
 

@Override
public void open(Configuration parameters) throws Exception {
	getRuntimeContext().addAccumulator(INCOMPATIBLE_ACCUMULATORS_NAME, new LongCounter());
}
 

@Override
public LongCounter clone() {
	return new FaultyMergeAccumulator();
}
 

@Override
public LongCounter getLongCounter(String name) {
	return (LongCounter) getAccumulator(name, LongCounter.class);
}
 

@Override
public LongCounter getLongCounter(final String name) {
	throw new UnsupportedOperationException("Long counters are not supported.");
}
 

public static void main(final String[] args) throws Exception {
	// set up the execution environment
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	String in = args[0];
	String out = args[1];
	System.err.println("Using input=" + in);
	System.err.println("Using output=" + out);

	String patterns[] = new String[args.length - 2];
	System.arraycopy(args, 2, patterns, 0, args.length - 2);
	System.err.println("Using patterns: " + Arrays.toString(patterns));

	// get input data
	DataSet<String> text = env.readTextFile(args[0]);
	for (int p = 0; p < patterns.length; p++) {
		final String pattern = patterns[p];
		DataSet<String> res = text.filter(new RichFilterFunction<String>() {
			private static final long serialVersionUID = 1L;

			Pattern p = Pattern.compile(pattern);
			LongCounter filterMatches = new LongCounter();
			LongCounter filterRecords = new LongCounter();

			@Override
			public void open(Configuration parameters) throws Exception {
				super.open(parameters);
				getRuntimeContext().addAccumulator("filterMatchCount-" + pattern, filterMatches);
				getRuntimeContext().addAccumulator("filterRecordCount-" + pattern, filterRecords);
			}

			@Override
			public boolean filter(String value) throws Exception {
				filterRecords.add(1L);
				if (value == null || value.length() == 0) {
					return false;
				}
				final Matcher m = p.matcher(value);
				if (m.find()) {
					filterMatches.add(1L);
					return true;
				}
				return false;
			}
		}).name("grep for " + pattern);
		res.writeAsText(out + "_" + pattern, FileSystem.WriteMode.OVERWRITE);
	}

	// execute program
	JobExecutionResult jobResult = env.execute("Flink Grep benchmark");
	System.err.println(AccumulatorHelper.getResultsFormated(jobResult.getAllAccumulatorResults()));
}