org.apache.hadoop.hive.metastore.api.PartitionSpec Java Examples

/**
 * {@inheritDoc}
 */
@Override
public int add_partitions_pspec(final List<PartitionSpec> newParts) throws TException {
    if (newParts == null || newParts.isEmpty()) {
        return 0;
    }
    final String dbName = newParts.get(0).getDbName();
    final String tableName = newParts.get(0).getTableName();
    return requestWrapper("add_partition", new Object[]{dbName, tableName}, () -> {
        final PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(newParts);
        final PartitionSpecProxy.PartitionIterator partitionIterator = partitionSpecProxy.getPartitionIterator();
        final List<Partition> partitions = addPartitionsCore(dbName, tableName,
            Lists.newArrayList(partitionIterator), false);
        return partitions.size();
    });
}
 

@Override
public List<PartitionSpec> transformOutboundPartitionSpecs(List<PartitionSpec> partitionSpecs) {
  for (PartitionSpec partitionSpec : partitionSpecs) {
    transformOutboundPartitionSpec(partitionSpec);
  }
  return partitionSpecs;
}
 

@Override
public List<PartitionSpec> transformInboundPartitionSpecs(List<PartitionSpec> partitionSpecs) {
  for (PartitionSpec partitionSpec : partitionSpecs) {
    partitionSpec.setDbName(metaStoreMapping.transformInboundDatabaseName(partitionSpec.getDbName()));
  }
  return partitionSpecs;
}
 

/**
 * {@inheritDoc}
 */
@Override
public List<PartitionSpec> get_part_specs_by_filter(final String dbName, final String tblName,
                                                    final String filter, final int maxParts)
    throws TException {
    //TODO: Handle the use case of grouping
    return requestWrapper("get_partitions_pspec", new Object[]{dbName, tblName, filter, maxParts}, () -> {
        final String databaseName = normalizeIdentifier(dbName);
        final String tableName = normalizeIdentifier(tblName);
        final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false);
        final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(filter, null, true, false);
        final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, dbName, tblName,
            null, null, null, maxParts, false, dto);
        final List<Partition> partitions = Lists.newArrayListWithCapacity(metacatPartitions.size());
        for (PartitionDto partition : metacatPartitions) {
            partitions.add(hiveConverters.metacatToHivePartition(partition, tableDto));
        }

        final PartitionSpec pSpec = new PartitionSpec();
        pSpec.setPartitionList(new PartitionListComposingSpec(partitions));
        pSpec.setDbName(dbName);
        pSpec.setTableName(tblName);
        if (tableDto != null && tableDto.getSerde() != null) {
            pSpec.setRootPath(tableDto.getSerde().getUri());
        }
        return Arrays.asList(pSpec);
    });
}
 

@Test
public void get_part_specs_by_filter() throws TException {
  List<PartitionSpec> partitionSpecs = Lists.newArrayList();
  List<PartitionSpec> outbound = Lists.newArrayList();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(primaryClient.get_part_specs_by_filter("inbound", "table", "*", (short) 10)).thenReturn(partitionSpecs);
  when(primaryMapping.transformOutboundPartitionSpecs(partitionSpecs)).thenReturn(outbound);
  List<PartitionSpec> result = handler.get_part_specs_by_filter(DB_P, "table", "*", (short) 10);
  assertThat(result, is(outbound));
  verify(primaryMapping, never()).checkWritePermissions(DB_P);
}
 

@Test
public void get_partitions_pspec() throws TException {
  List<PartitionSpec> partitionSpecs = Lists.newArrayList();
  List<PartitionSpec> outbound = Lists.newArrayList();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(primaryClient.get_partitions_pspec("inbound", "table", (short) 10)).thenReturn(partitionSpecs);
  when(primaryMapping.transformOutboundPartitionSpecs(partitionSpecs)).thenReturn(outbound);
  List<PartitionSpec> result = handler.get_partitions_pspec(DB_P, "table", (short) 10);
  assertThat(result, is(outbound));
  verify(primaryMapping, never()).checkWritePermissions(DB_P);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public int add_partitions_pspec(List<PartitionSpec> new_parts)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  if (!new_parts.isEmpty()) {
    // Need to pick one mapping and use that for permissions and getting the client.
    // If the partitions added are for different databases in different clients that won't work with waggle-dance
    DatabaseMapping mapping = databaseMappingService.databaseMapping(new_parts.get(0).getDbName());
    for (PartitionSpec partitionSpec : new_parts) {
      mapping.checkWritePermissions(partitionSpec.getDbName());
    }
    return mapping.getClient().add_partitions_pspec(mapping.transformInboundPartitionSpecs(new_parts));
  }
  return 0;
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME, prepend=true)
public List<PartitionSpec> get_partitions_pspec(String db_name, String tbl_name, int max_parts)
    throws NoSuchObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  List<PartitionSpec> partitionSpecs = mapping
      .getClient()
      .get_partitions_pspec(mapping.transformInboundDatabaseName(db_name), tbl_name, max_parts);
  return mapping.transformOutboundPartitionSpecs(partitionSpecs);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME, prepend=true)
public List<PartitionSpec> get_part_specs_by_filter(String db_name, String tbl_name, String filter, int max_parts)
    throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  List<PartitionSpec> partitionSpecs = mapping
      .getClient()
      .get_part_specs_by_filter(mapping.transformInboundDatabaseName(db_name), tbl_name, filter, max_parts);
  return mapping.transformOutboundPartitionSpecs(partitionSpecs);
}
 

@Test
public void add_partitions_pspec() throws TException {
  PartitionSpec newPartitionPSpec1 = new PartitionSpec();
  newPartitionPSpec1.setDbName(DB_P);
  PartitionSpec newPartitionPspec2 = new PartitionSpec();
  newPartitionPspec2.setDbName(DB_P);
  List<PartitionSpec> inbound = Lists.newArrayList(new PartitionSpec());
  List<PartitionSpec> partitionsPspec = Lists.newArrayList(newPartitionPSpec1, newPartitionPspec2);
  when(primaryMapping.transformInboundPartitionSpecs(partitionsPspec)).thenReturn(inbound);
  when(primaryClient.add_partitions_pspec(inbound)).thenReturn(2);
  int result = handler.add_partitions_pspec(partitionsPspec);
  assertThat(result, is(2));
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
}
 

@Test
public void transformInboundPartitionSpecs() throws Exception {
  List<PartitionSpec> partitionSpecs = Lists.newArrayList(partitionSpec);
  List<PartitionSpec> result = databaseMapping.transformInboundPartitionSpecs(partitionSpecs);
  assertThat(result, is(sameInstance(partitionSpecs)));
  PartitionSpec resultSpec = result.get(0);
  assertThat(resultSpec, is(sameInstance(partitionSpec)));
  assertThat(resultSpec.getDbName(), is(IN_DB_NAME));
}
 

@Test
public void transformOutboundPartitionSpecs() throws Exception {
  List<PartitionSpec> partitionSpecs = new ArrayList<>();
  partitionSpecs.add(partitionSpec);
  List<PartitionSpec> result = databaseMapping.transformOutboundPartitionSpecs(partitionSpecs);
  assertThat(result, is(sameInstance(partitionSpecs)));
  PartitionSpec resultSpec = result.get(0);
  assertThat(resultSpec, is(sameInstance(partitionSpec)));
  assertThat(resultSpec.getDbName(), is(OUT_DB_NAME));
}
 

@Before
public void setUp() {
  databaseMapping = new DatabaseMappingImpl(metastoreMapping, queryMapping);
  database = new Database();
  database.setName(DB_NAME);
  partition = new Partition();
  partition.setDbName(DB_NAME);
  partitions = Lists.newArrayList(partition);
  index = new Index();
  index.setDbName(DB_NAME);
  hiveObjectRef = new HiveObjectRef();
  hiveObjectRef.setDbName(DB_NAME);
  hiveObjectRef.setObjectType(HiveObjectType.DATABASE);
  hiveObjectRef.setObjectName(DB_NAME);
  hiveObjectPrivileges = new ArrayList<>();
  HiveObjectPrivilege hiveObjectPrivilege = new HiveObjectPrivilege();
  hiveObjectPrivilege.setHiveObject(hiveObjectRef);
  hiveObjectPrivileges.add(hiveObjectPrivilege);
  partitionSpec = new PartitionSpec();
  partitionSpec.setDbName(DB_NAME);
  when(metastoreMapping.transformInboundDatabaseName(anyString())).thenReturn(IN_DB_NAME);
  when(metastoreMapping.transformOutboundDatabaseName(anyString())).thenReturn(OUT_DB_NAME);
  when(queryMapping.transformOutboundDatabaseName(metastoreMapping, VIEW_EXPANDED_TEXT))
      .thenReturn(VIEW_EXPANDED_TEXT_TRANSFORMED);
  when(queryMapping.transformOutboundDatabaseName(metastoreMapping, VIEW_ORIGINAL_TEXT))
      .thenReturn(VIEW_ORIGINAL_TEXT_TRANSFORMED);
}
 

@Override
public List<PartitionSpec> filterPartitionSpecs(
    List<PartitionSpec> partitionSpecList) {
  return partitionSpecList;
}
 

@Override
public PartitionSpec transformOutboundPartitionSpec(PartitionSpec partitionSpec) {
  partitionSpec.setDbName(metaStoreMapping.transformOutboundDatabaseName(partitionSpec.getDbName()));
  return partitionSpec;
}
 

/**
 * {@inheritDoc}
 */
@Override
public List<PartitionSpec> get_partitions_pspec(final String dbName, final String tblName, final int maxParts)
    throws TException {
    return get_part_specs_by_filter(dbName, tblName, null, maxParts);
}
 

@Test
public void transformOutboundPartitionSpec() throws Exception {
  PartitionSpec result = databaseMapping.transformOutboundPartitionSpec(partitionSpec);
  assertThat(result, is(sameInstance(partitionSpec)));
  assertThat(result.getDbName(), is(OUT_DB_NAME));
}
 

@Test
public void transformInboundPartitionSpecs() throws Exception {
  List<PartitionSpec> partitionSpecs = new ArrayList<>();
  List<PartitionSpec> result = databaseMapping.transformInboundPartitionSpecs(partitionSpecs);
  assertThat(result, is(sameInstance(partitionSpecs)));
}
 

@Test
public void transformOutboundPartitionSpecs() throws Exception {
  List<PartitionSpec> partitionSpecs = new ArrayList<>();
  List<PartitionSpec> result = databaseMapping.transformOutboundPartitionSpecs(partitionSpecs);
  assertThat(result, is(sameInstance(partitionSpecs)));
}
 

@Test
public void transformOutboundPartitionSpec() throws Exception {
  PartitionSpec partitionSpec = new PartitionSpec();
  PartitionSpec result = databaseMapping.transformOutboundPartitionSpec(partitionSpec);
  assertThat(result, is(sameInstance(partitionSpec)));
}
 

@Override
public List<PartitionSpec> transformInboundPartitionSpecs(List<PartitionSpec> partitionSpecs) {
  return partitionSpecs;
}
 

@Override
public List<PartitionSpec> transformOutboundPartitionSpecs(List<PartitionSpec> partitionSpecs) {
  return partitionSpecs;
}
 

@Override
public PartitionSpec transformOutboundPartitionSpec(PartitionSpec request) {
  return request;
}
 

List<PartitionSpec> transformInboundPartitionSpecs(List<PartitionSpec> partitionSpecs); 

List<PartitionSpec> transformOutboundPartitionSpecs(List<PartitionSpec> partitionSpecs); 

PartitionSpec transformOutboundPartitionSpec(PartitionSpec partitionSpec);