org.apache.hive.hcatalog.data.transfer.DataTransferFactory Java Examples

private static ReaderContext getHiveReaderContext(String database, String table, Map<String, String> partitionKV) throws Exception {
    HiveConf hiveConf = new HiveConf(HiveTableReader.class);
    Iterator<Entry<String, String>> itr = hiveConf.iterator();
    Map<String, String> map = new HashMap<String, String>();
    while (itr.hasNext()) {
        Entry<String, String> kv = itr.next();
        map.put(kv.getKey(), kv.getValue());
    }

    ReadEntity entity;
    if (partitionKV == null || partitionKV.size() == 0) {
        entity = new ReadEntity.Builder().withDatabase(database).withTable(table).build();
    } else {
        entity = new ReadEntity.Builder().withDatabase(database).withTable(table).withPartition(partitionKV).build();
    }

    HCatReader reader = DataTransferFactory.getHCatReader(entity, map);
    ReaderContext cntxt = reader.prepareRead();

    return cntxt;
}
 

private static ReaderContext getHiveReaderContext(String database, String table, Map<String, String> partitionKV) throws Exception {
    HiveConf hiveConf = new HiveConf(HiveTableReader.class);
    Iterator<Entry<String, String>> itr = hiveConf.iterator();
    Map<String, String> map = new HashMap<String, String>();
    while (itr.hasNext()) {
        Entry<String, String> kv = itr.next();
        map.put(kv.getKey(), kv.getValue());
    }

    ReadEntity entity;
    if (partitionKV == null || partitionKV.size() == 0) {
        entity = new ReadEntity.Builder().withDatabase(database).withTable(table).build();
    } else {
        entity = new ReadEntity.Builder().withDatabase(database).withTable(table).withPartition(partitionKV).build();
    }

    HCatReader reader = DataTransferFactory.getHCatReader(entity, map);
    ReaderContext cntxt = reader.prepareRead();

    return cntxt;
}
 

private static ReaderContext getHiveReaderContext(String database, String table, Map<String, String> partitionKV) throws Exception {
    HiveConf hiveConf = new HiveConf(HiveTableReader.class);
    Iterator<Entry<String, String>> itr = hiveConf.iterator();
    Map<String, String> map = new HashMap<String, String>();
    while (itr.hasNext()) {
        Entry<String, String> kv = itr.next();
        map.put(kv.getKey(), kv.getValue());
    }

    ReadEntity entity;
    if (partitionKV == null || partitionKV.size() == 0) {
        entity = new ReadEntity.Builder().withDatabase(database).withTable(table).build();
    } else {
        entity = new ReadEntity.Builder().withDatabase(database).withTable(table).withPartition(partitionKV).build();
    }

    HCatReader reader = DataTransferFactory.getHCatReader(entity, map);
    ReaderContext cntxt = reader.prepareRead();

    return cntxt;
}
 

private void insert(Map<String, String> partitionSpec, Iterable<HCatRecord> rows) {
  WriteEntity entity = new WriteEntity.Builder()
      .withDatabase(databaseName)
      .withTable(tableName)
      .withPartition(partitionSpec)
      .build();

  try {
    HCatWriter master = DataTransferFactory.getHCatWriter(entity, config);
    WriterContext context = master.prepareWrite();
    HCatWriter writer = DataTransferFactory.getHCatWriter(context);
    writer.write(rows.iterator());
    master.commit(context);
  } catch (HCatException e) {
    throw new RuntimeException("An error occurred while inserting data to " + databaseName + "." + tableName, e);
  }
}
 

private ReaderContext getReaderContext(Read readRequest, Integer partitionIndexToRead)
    throws Exception {
  final List<Partition> partitions =
      metaStoreClient.listPartitions(
          readRequest.getDatabase(), readRequest.getTable(), Short.MAX_VALUE);
  final Partition partition = partitions.get(partitionIndexToRead);
  checkArgument(
      partition != null, "Unable to find a partition to read at index " + partitionIndexToRead);

  final int desiredSplitCount = HCatalogUtils.getSplitCount(readRequest, partition);
  final List<String> values = partition.getValues();
  final List<String> partitionCols = readRequest.getPartitionCols();
  checkArgument(
      values.size() == partitionCols.size(),
      "Number of input partitions should be equal to the values of list partition values.");

  List<String> filter = new ArrayList<>();
  for (int i = 0; i < partitionCols.size(); i++) {
    filter.add(partitionCols.get(i) + "=" + "'" + values.get(i) + "'");
  }
  final String filterString = String.join(" and ", filter);

  ReadEntity entity =
      new ReadEntity.Builder()
          .withDatabase(readRequest.getDatabase())
          .withFilter(filterString)
          .withTable(readRequest.getTable())
          .build();
  // pass the 'desired' split count as an hint to the API
  Map<String, String> configProps = new HashMap<>(readRequest.getConfigProperties());
  configProps.put(
      HCatConstants.HCAT_DESIRED_PARTITION_NUM_SPLITS, String.valueOf(desiredSplitCount));
  return DataTransferFactory.getHCatReader(entity, configProps).prepareRead();
}
 

@ProcessElement
public void processElement(ProcessContext c) throws Exception {
  final Read readRequest = c.element().getKey();
  final Integer partitionIndexToRead = c.element().getValue();
  ReaderContext readerContext = getReaderContext(readRequest, partitionIndexToRead);
  for (int i = 0; i < readerContext.numSplits(); i++) {
    HCatReader reader = DataTransferFactory.getHCatReader(readerContext, i);
    Iterator<HCatRecord> hcatIterator = reader.read();
    while (hcatIterator.hasNext()) {
      final HCatRecord record = hcatIterator.next();
      c.output(record);
    }
  }
}
 

private ReaderContext getReaderContext(long desiredSplitCount) throws HCatException {
  ReadEntity entity =
      new ReadEntity.Builder()
          .withDatabase(spec.getDatabase())
          .withTable(spec.getTable())
          .withFilter(spec.getFilter())
          .build();
  // pass the 'desired' split count as an hint to the API
  Map<String, String> configProps = new HashMap<>(spec.getConfigProperties());
  configProps.put(
      HCatConstants.HCAT_DESIRED_PARTITION_NUM_SPLITS, String.valueOf(desiredSplitCount));
  return DataTransferFactory.getHCatReader(entity, configProps).prepareRead();
}
 

@Override
public boolean start() throws HCatException {
  HCatReader reader =
      DataTransferFactory.getHCatReader(source.spec.getContext(), source.spec.getSplitId());
  hcatIterator = reader.read();
  return advance();
}
 

@Setup
public void initiateWrite() throws HCatException {
  WriteEntity entity =
      new WriteEntity.Builder()
          .withDatabase(spec.getDatabase())
          .withTable(spec.getTable())
          .withPartition(spec.getPartition())
          .build();
  masterWriter = DataTransferFactory.getHCatWriter(entity, spec.getConfigProperties());
  writerContext = masterWriter.prepareWrite();
  slaveWriter = DataTransferFactory.getHCatWriter(writerContext);
}
 

private static Iterator<HCatRecord> loadHCatRecordItr(ReaderContext readCntxt, int dataSplit) throws HCatException {
    HCatReader currentHCatReader = DataTransferFactory.getHCatReader(readCntxt, dataSplit);

    return currentHCatReader.read();
}
 

/** Returns a ReaderContext instance for the passed datastore config params. */
public static ReaderContext getReaderContext(Map<String, String> config) throws HCatException {
  return DataTransferFactory.getHCatReader(READ_ENTITY, config).prepareRead();
}
 

/** Returns a WriterContext instance for the passed datastore config params. */
private static WriterContext getWriterContext(Map<String, String> config) throws HCatException {
  return DataTransferFactory.getHCatWriter(WRITE_ENTITY, config).prepareWrite();
}
 

/** Writes records to the table using the passed WriterContext. */
private static void writeRecords(WriterContext context) throws HCatException {
  DataTransferFactory.getHCatWriter(context)
      .write(buildHCatRecords(TEST_RECORDS_COUNT).iterator());
}
 

/** Commits the pending writes to the database. */
private static void commitRecords(Map<String, String> config, WriterContext context)
    throws IOException {
  DataTransferFactory.getHCatWriter(WRITE_ENTITY, config).commit(context);
}
 

private static Iterator<HCatRecord> loadHCatRecordItr(ReaderContext readCntxt, int dataSplit) throws HCatException {
    HCatReader currentHCatReader = DataTransferFactory.getHCatReader(readCntxt, dataSplit);

    return currentHCatReader.read();
}
 

private static Iterator<HCatRecord> loadHCatRecordItr(ReaderContext readCntxt, int dataSplit) throws HCatException {
    HCatReader currentHCatReader = DataTransferFactory.getHCatReader(readCntxt, dataSplit);
    return currentHCatReader.read();
}