cascading.tuple.TupleEntryIterator Java Examples

/**
 * Reads the {@link Tuple Tuples} from the {@link Tap} and returns them wrapped in a {@link Data} instance whose
 * {@link Fields} confirm to those supplied by {@link Tap#getSourceFields()}.
 */
Data read() throws IOException {
  TupleEntryIterator tuples = null;
  try {
    Class<?> tapConfigClass = TapTypeUtil.getTapConfigClass(source);

    if (Configuration.class.equals(tapConfigClass)) {
      tuples = getHadoopTupleEntryIterator();
    } else if (Properties.class.equals(tapConfigClass)) {
      tuples = getLocalTupleEntryIterator();
    } else {
      throw new IllegalArgumentException("Unsupported tap type: " + source.getClass());
    }
    List<Tuple> resultTuples = new ArrayList<Tuple>();
    while (tuples.hasNext()) {
      Tuple copy = new Tuple(tuples.next().getTupleCopy());
      resultTuples.add(copy);
    }
    return new Data(source.getSourceFields(), Collections.unmodifiableList(resultTuples));
  } finally {
    if (tuples != null) {
      tuples.close();
    }
  }
}
 

private TupleEntryIterator getHadoopTupleEntryIterator() throws IOException {
  @SuppressWarnings("unchecked")
  Tap<JobConf, ?, ?> hadoopTap = (Tap<JobConf, ?, ?>) source;
  JobConf conf = new JobConf();
  FlowProcess<JobConf> flowProcess = new HadoopFlowProcess(conf);
  hadoopTap.sourceConfInit(flowProcess, conf);
  return hadoopTap.openForRead(flowProcess);
}
 

private TupleEntryIterator getLocalTupleEntryIterator() throws IOException {
  @SuppressWarnings("unchecked")
  Tap<Properties, ?, ?> localTap = (Tap<Properties, ?, ?>) source;
  Properties properties = new Properties();
  FlowProcess<Properties> flowProcess = new LocalFlowProcess(properties);
  localTap.sourceConfInit(flowProcess, properties);
  return localTap.openForRead(flowProcess);
}
 

@Test
public void openForRead() throws IOException {
  TupleEntryIterator iterator = tap.openForRead(null, null);
  assertThat(iterator.hasNext(), is(true));
  assertThat(iterator.next(), is(new TupleEntry(FIELDS, TUPLE_1)));
  assertThat(iterator.hasNext(), is(true));
  assertThat(iterator.next(), is(new TupleEntry(FIELDS, TUPLE_2)));
  assertThat(iterator.hasNext(), is(false));
}
 

@Override
public TupleEntryIterator openTapForRead(Tap tap) throws IOException {
	return tap.openForRead( this );
}
 

/**
 * {@inheritDoc}
 * <p/>
 * Returned type is a {@link ListTupleEntryIterator}.
 */
@Override
public TupleEntryIterator openForRead(FlowProcess<? extends Properties> flowProcess, Iterator<Tuple> input)
  throws IOException {
  return new ListTupleEntryIterator(getSourceFields(), this.input);
}
 

/**
 * Always throws {@link UnsupportedOperationException} - this is a sink not a tap.
 *
 * @throws UnsupportedOperationException always.
 */
@Override
public TupleEntryIterator openForRead(FlowProcess<? extends Properties> flowProcess, Iterator<Tuple> input)
    throws IOException {
  throw new UnsupportedOperationException("cannot read from a " + getClass().getSimpleName());
}
 

@Override
public TupleEntryIterator openForRead(FlowProcess<? extends String> flowProcess, Long input) throws IOException {
  return null;
}
 

@Override
public TupleEntryIterator openForRead( FlowProcess<JobConf> flowProcess, RecordReader input ) throws IOException {
    // input may be null when this method is called on the client side or cluster side when accumulating
    // for a HashJoin
    return new HadoopTupleEntrySchemeIterator( flowProcess, this, input );
}
 

@Override
public TupleEntryIterator openForRead(FlowProcess<JobConf> jobConfFlowProcess, RecordReader recordReader) throws IOException {
  return new HadoopTupleEntrySchemeIterator(jobConfFlowProcess, this, recordReader);
}
 

@Override
public TupleEntryIterator openForRead(FlowProcess<JobConf> jobConfFlowProcess, RecordReader recordReader)
		throws IOException {
	return new HadoopTupleEntrySchemeIterator(jobConfFlowProcess, this, recordReader);
}