Java Code Examples for com.google.cloud.dataflow.sdk.values.KV#getKey()

@Override
public KV<String, WorkflowArgs> apply(Iterable<KV<String, WorkflowArgs>> input) {
  String key = null;
  WorkflowArgs retval = null;

  // Merge arguments
  for (KV<String, WorkflowArgs> kv : input) {

    // Modify a copy
    WorkflowArgs wa = new WorkflowArgs(kv.getValue());

    // First time, nothing to merge
    if (retval == null) {
      key = kv.getKey();
      retval = wa;
    // Find differences and merge
    } else {
      retval.gatherArgs(wa);
    }
  }
  return KV.of(key, retval);
}
 

@Override
public void combine(Iterable<KV<K, VI>> elements, Collector<KV<K, VA>> out) throws Exception {

	final Iterator<KV<K, VI>> iterator = elements.iterator();
	// create accumulator using the first elements key
	KV<K, VI> first = iterator.next();
	K key = first.getKey();
	VI value = first.getValue();
	VA accumulator = keyedCombineFn.createAccumulator(key);
	accumulator = keyedCombineFn.addInput(key, accumulator, value);

	while(iterator.hasNext()) {
		value = iterator.next().getValue();
		accumulator = keyedCombineFn.addInput(key, accumulator, value);
	}

	out.collect(KV.of(key, accumulator));
}
 

/**
 * Converts a Key-Value pair of (FilePath, FileContent) to (GCPResource, GCPResourceState).
 * The FilePath is a list of Strings which represents the location of a file.
 * The FileContent is the content of the file described by the FilePath.
 * The path is used to obtain the resource, and the content describes the state of that resource.
 * @param processContext The ProcessContext object that contains processContext-specific
 * methods and objects.
 */
@Override
public void processElement(ProcessContext processContext) {
  KV<List<String>, String> input = processContext.element();
  List<String> filePath = input.getKey();
  String fileContent = input.getValue();

  String orgName = filePath.size() > 0 ? filePath.get(0) : null;
  String projectId = filePath.size() > 1 ? filePath.get(1) : null;
  String policyFileName = filePath.size() > 2 ? filePath.get(2) : null;
  GCPProject project = new GCPProject(projectId, orgName);

  if (filePath.size() == 3 && GCPResourcePolicy.getPolicyFile().equals(policyFileName)) {
    // only project policies are supported for now.
    // filePath.size() must be 3 and of the form org_id/project_id/POLICY_FILE.
    Gson gson = new Gson();
    try {
      List<PolicyBinding> bindings = Arrays.asList(
          gson.fromJson(fileContent, PolicyBinding[].class));
      GCPResourceState policy = new GCPResourcePolicy(project, bindings);
      processContext.output(KV.of((GCPResource) project, policy));
      return;
    } catch (JsonSyntaxException jse) {
      addToSideOutput(
          processContext,
          project,
          String.format("Invalid policy json %s/%s/%s", orgName, projectId, policyFileName));
    }
  }
  addToSideOutput(
      processContext,
      project,
      String.format("Invalid policy filepath %s/%s/%s", orgName, projectId, policyFileName));
}
 

@Override
public int hash(KV<K, V> record) {
	K key = record.getKey();
	if (key != null) {
		return key.hashCode();
	} else {
		return 0;
	}
}
 

@Override
public int extractKeys(Object record, Object[] target, int index) {
	KV<K, V> kv = (KV<K, V>) record;
	K k = kv.getKey();
	target[index] = k;
	return 1;
}
 

@Override
public void reduce(Iterable<KV<K, VA>> values, Collector<KV<K, VO>> out) throws Exception {
	Iterator<KV<K, VA>> it = values.iterator();

	KV<K, VA> current = it.next();
	K k = current.getKey();
	VA accumulator = current.getValue();

	while (it.hasNext()) {
		current = it.next();
		keyedCombineFn.mergeAccumulators(k, ImmutableList.of(accumulator, current.getValue()) );
	}

	out.collect(KV.of(k, keyedCombineFn.extractOutput(k, accumulator)));
}
 

@Override
public void processElement(ProcessContext c) {
	KV<Integer, String> record = c.element();
	long now = System.currentTimeMillis();
	int timestamp = record.getKey();
	String userName = record.getValue();
	if (userName != null) {
		// Sets the implicit timestamp field to be used in windowing.
		c.outputWithTimestamp(userName, new Instant(timestamp + now));
	}
}
 

@Override
public String apply(KV<String, Long> input) {
	return input.getKey() + ": " + input.getValue();
}