Java Code Examples for cascading.tuple.TupleEntryCollector#add()

The following examples show how to use cascading.tuple.TupleEntryCollector#add() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TapDataWriter.java    From plunger with Apache License 2.0 6 votes vote down vote up 
private void writeToHadoopPartitionTap(Tap<?, ?, ?> tap) throws IOException {
  @SuppressWarnings("unchecked")
  BasePartitionTap<JobConf, ?, ?> hadoopTap = (BasePartitionTap<JobConf, ?, ?>) tap;
  JobConf conf = new JobConf();

  // Avoids deletion of results when using a partition tap (close() will delete the _temporary before the copy has
  // been done if not in a flow)
  HadoopUtil.setIsInflow(conf);

  HadoopFlowProcess flowProcess = new HadoopFlowProcess(conf);
  hadoopTap.sinkConfInit(flowProcess, conf);
  TupleEntryCollector collector = hadoopTap.openForWrite(flowProcess);
  for (TupleEntry tuple : data.asTupleEntryList()) {
    collector.add(tuple);
  }
  collector.close();

  // We need to clean up the '_temporary' folder
  BasePartitionTap<JobConf, ?, ?> partitionTap = hadoopTap;
  @SuppressWarnings("unchecked")
  String basePath = partitionTap.getParent().getFullIdentifier(flowProcess);
  deleteTemporaryPath(new Path(basePath), FileSystem.get(conf));
}
Example 2
Source File: TapDataWriter.java    From plunger with Apache License 2.0 6 votes vote down vote up 
private void writeToLocalTap(Tap<?, ?, ?> tap) throws IOException {
  @SuppressWarnings("unchecked")
  Tap<Properties, ?, ?> localTap = (Tap<Properties, ?, ?>) tap;
  Properties conf = new Properties();
  LocalFlowProcess flowProcess = new LocalFlowProcess(conf);

  flowProcess.setStepStats(new LocalStepStats(new NullFlowStep(), NullClientState.INSTANCE));

  localTap.sinkConfInit(flowProcess, conf);
  TupleEntryCollector collector = localTap.openForWrite(flowProcess);
  for (TupleEntry tuple : data.asTupleEntryList()) {
    collector.add(tuple);
  }
  collector.close();
  localTap.commitResource(conf);
}
Example 3
Source File: TapDataWriter.java    From plunger with Apache License 2.0 5 votes vote down vote up 
private void writeToHadoopTap(Tap<?, ?, ?> tap) throws IOException {
  @SuppressWarnings("unchecked")
  Tap<JobConf, ?, ?> hadoopTap = (Tap<JobConf, ?, ?>) tap;
  JobConf conf = new JobConf();

  HadoopFlowProcess flowProcess = new HadoopFlowProcess(conf);
  hadoopTap.sinkConfInit(flowProcess, conf);
  TupleEntryCollector collector = hadoopTap.openForWrite(flowProcess);
  for (TupleEntry tuple : data.asTupleEntryList()) {
    collector.add(tuple);
  }
  collector.close();
}
Example 4
Source File: BucketTest.java    From plunger with Apache License 2.0 5 votes vote down vote up 
@Test
public void asTupleEntryList() throws IOException {
  Bucket sink = new Bucket(FIELDS, pipe, flow);
  TupleEntryCollector collector = sink.openForWrite(null, null);
  collector.add(TUPLE_1);
  collector.add(TUPLE_2);
  List<TupleEntry> tupleEntryList = sink.result().asTupleEntryList();
  assertThat(tupleEntryList.size(), is(2));
  assertThat(tupleEntryList.get(0).getFields(), is(FIELDS));
  assertThat(tupleEntryList.get(0).getTuple(), is(TUPLE_1));
  assertThat(tupleEntryList.get(1).getFields(), is(FIELDS));
  assertThat(tupleEntryList.get(1).getTuple(), is(TUPLE_2));
}
Example 5
Source File: BucketTest.java    From plunger with Apache License 2.0 5 votes vote down vote up 
@Test
public void asTupleList() throws IOException {
  Bucket sink = new Bucket(FIELDS, pipe, flow);
  TupleEntryCollector collector = sink.openForWrite(null, null);
  collector.add(TUPLE_1);
  collector.add(TUPLE_2);
  List<Tuple> tupleList = sink.result().asTupleList();
  assertThat(tupleList.size(), is(2));
  assertThat(tupleList.get(0), is(TUPLE_1));
  assertThat(tupleList.get(1), is(TUPLE_2));
}