Java Code Examples for com.google.cloud.dataflow.sdk.values.KV#of()

/**
 * Get the next file in queue.
 * @return A Key-Value pair where the key is a list of strings representing the path of
 * the file and the value is a string representing the content of the file.
 * @throws NoSuchElementException If the file can't be read from the GCS API.
 */
@Override
public KV<List<String>, String> getCurrent() throws NoSuchElementException {
  String filePath = this.currentFiles.get(0);
  String fileContent = null;
  try {
    fileContent = this.source.getFileContent(filePath);
  } catch (IOException ioe) {
    throw new NoSuchElementException(
        "Object " + filePath + " not found in bucket " + this.source.bucket);
  } catch (GeneralSecurityException gse) {
    throw new NoSuchElementException(
        "Cannot access object "
            + filePath
            + " in bucket "
            + this.source.bucket
            + " due to security reasons");
  }
  List<String> splitPath = Arrays.asList(filePath.split(this.source.getDirDelimiter()));
  return KV.of(splitPath, fileContent);
}
 

@Override
public KV<String, WorkflowArgs> apply(Iterable<KV<String, WorkflowArgs>> input) {
  String key = null;
  WorkflowArgs retval = null;

  // Merge arguments
  for (KV<String, WorkflowArgs> kv : input) {

    // Modify a copy
    WorkflowArgs wa = new WorkflowArgs(kv.getValue());

    // First time, nothing to merge
    if (retval == null) {
      key = kv.getKey();
      retval = wa;
    // Find differences and merge
    } else {
      retval.gatherArgs(wa);
    }
  }
  return KV.of(key, retval);
}
 

@Test
public void doMutation_encodesKeysAndCounts() {
  // Arrange
  DoFnTester<KV<String, Integer>, Mutation> tester = DoFnTester.of(LoadBooks.ENCODE_NGRAM);
  KV<String, Integer> input = KV.of("this is a test", 513);

  // Act
  List<Mutation> output = tester.processBatch(input);

  // Assert
  Put put = (Put) output.get(0);
  assertThat(put.getRow()).isEqualTo("this is a test".getBytes(StandardCharsets.UTF_8));
  Cell valueCell = put.get(LoadBooks.FAMILY, LoadBooks.COUNT_QUALIFIER).get(0);
  byte[] valueArray = valueCell.getValueArray();
  byte[] value =
      Arrays.copyOfRange(
          valueArray,
          valueCell.getValueOffset(),
          valueCell.getValueOffset() + valueCell.getValueLength());
  assertThat(value).isEqualTo(new byte[] {0, 0, 2, 1});
}
 

@Override
public KV<LatLon, TableRow> apply(TableRow t) {
  float lat = Float.parseFloat(t.get("latitude").toString());
  float lon = Float.parseFloat(t.get("longitude").toString());
  final float PRECISION = 0.005f; // very approximately 500m
  float roundedLat = (float) Math.floor(lat / PRECISION) * PRECISION + PRECISION / 2;
  float roundedLon = (float) Math.floor(lon / PRECISION) * PRECISION + PRECISION / 2;
  LatLon key = new LatLon(roundedLat, roundedLon);

  return KV.of(key, t);
}
 

@Override
  public void processElement(ProcessContext c) {
  String word = c.element().value;
  for (int i = minPrefix; i <= Math.min(word.length(), maxPrefix); i++) {
    KV<String, CompletionCandidate> kv = KV.of(word.substring(0, i), c.element());
    c.output(kv);
  }
}
 

@Override
public void processElement(
    DoFn<KV<String, TSAggValueProto>, KV<String, WorkDataPoint>>.ProcessContext c) 
    throws Exception {

  // Extract only the needed data to do the work and create Proto

  TSProto openFX = c.element().getValue().getOpenState();
  TSProto closeFX = c.element().getValue().getCloseState();

  double closeOverOpen = closeFX.getAskPrice() / openFX.getAskPrice();

  double logRtn = Math.log(closeOverOpen);
  
  String key = c.element().getKey();

  WorkDataPoint data =
      WorkDataPoint.newBuilder().setKey(key).setTime(closeFX.getTime())
          .setValue(logRtn).build();

    String partition = c.window().maxTimestamp().toString();
    
    KV<String, WorkDataPoint> bars = KV.of(partition, data);
    
    c.outputWithTimestamp(bars, c.timestamp());

    c.sideOutput(tag, 1);
}