Java Code Examples for org.apache.hadoop.hive.metastore.api.Partition#setValues()

private void loadPartition(Path srcDir, Table table, Map<String, String> partSpec, HiveMetastoreClientWrapper client)
		throws TException, IOException {
	Path tblLocation = new Path(table.getSd().getLocation());
	String dbName = tablePath.getDatabaseName();
	String tableName = tablePath.getObjectName();
	List<Partition> existingPart = client.listPartitions(dbName, tableName, new ArrayList<>(partSpec.values()), (short) 1);
	Path destDir = existingPart.isEmpty() ? new Path(tblLocation, Warehouse.makePartPath(partSpec)) :
			new Path(existingPart.get(0).getSd().getLocation());
	moveFiles(srcDir, destDir);
	// register new partition if it doesn't exist
	if (existingPart.isEmpty()) {
		StorageDescriptor sd = new StorageDescriptor(hiveTablePartition.getStorageDescriptor());
		sd.setLocation(destDir.toString());
		Partition partition = HiveTableUtil.createHivePartition(dbName, tableName,
				new ArrayList<>(partSpec.values()), sd, new HashMap<>());
		partition.setValues(new ArrayList<>(partSpec.values()));
		client.add_partition(partition);
	}
}
 

@Test
public void dropReplacedPartitionsTest() throws Exception {

  Table table = ConvertibleHiveDatasetTest.getTestTable("dbName", "tableName");
  table.setTableType("VIRTUAL_VIEW");
  table.setPartitionKeys(ImmutableList.of(new FieldSchema("year", "string", ""), new FieldSchema("month", "string", "")));

  Partition part = new Partition();
  part.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01"));

  SchemaAwareHiveTable hiveTable = new SchemaAwareHiveTable(table, null);
  SchemaAwareHivePartition partition = new SchemaAwareHivePartition(table, part, null);

  QueryBasedHiveConversionEntity conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(partition));
  List<ImmutableMap<String, String>> expected =
      ImmutableList.of(ImmutableMap.of("year", "2015", "month", "12"), ImmutableMap.of("year", "2016", "month", "01"));
  Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected);

  // Make sure that a partition itself is not dropped
  Partition replacedSelf = new Partition();
  replacedSelf.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01|2016,02"));
  replacedSelf.setValues(ImmutableList.of("2016", "02"));

  conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(new SchemaAwareHivePartition(table, replacedSelf, null)));
  Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected);
}
 

/**
 * Convert a {@link HivePartition} into a {@link Partition}.
 */
public static Partition getPartition(HivePartition hivePartition) {
  State props = hivePartition.getProps();
  Partition partition = new Partition();
  partition.setDbName(hivePartition.getDbName());
  partition.setTableName(hivePartition.getTableName());
  partition.setValues(hivePartition.getValues());
  partition.setParameters(getParameters(props));
  if (hivePartition.getCreateTime().isPresent()) {
    partition.setCreateTime(Ints.checkedCast(hivePartition.getCreateTime().get()));
  } else if (props.contains(HiveConstants.CREATE_TIME)) {
    partition.setCreateTime(props.getPropAsInt(HiveConstants.CREATE_TIME));
  }
  if (props.contains(HiveConstants.LAST_ACCESS_TIME)) {
    partition.setLastAccessTime(props.getPropAsInt(HiveConstants.LAST_ACCESS_TIME));
  }
  partition.setSd(getStorageDescriptor(hivePartition));
  return partition;
}
 

@Test
public void dropDeletedPartitionsNothingToDrop() throws Exception {
  when(client.getTable(DATABASE, REPLICA_TABLE)).thenReturn(table);
  Partition replicaPartition = new Partition();
  replicaPartition.setValues(Lists.newArrayList("value1"));

  List<Partition> replicaPartitions = Lists.newArrayList(replicaPartition);
  mockPartitionIterator(replicaPartitions);

  // source partition ("part1=value1") is the same as target partition nothing is deleted on source, nothing should be
  // deleted on target
  List<String> sourcePartitionNames = Lists.newArrayList("part1=value1");
  replica.dropDeletedPartitions(sourcePartitionNames);
  verify(client, never()).dropPartition(DATABASE, REPLICA_TABLE, "part1=value1", false);
  verify(cleanupLocationManager, never()).addCleanupLocation(anyString(), any(Path.class));
  verify(client).close();
  verify(cleanupLocationManager).scheduleLocations();
}
 

@Test
public void newReplicaPartitionStatistics() throws MetaException {
  sourceTable.setPartitionKeys(
      Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null)));

  Partition replicaPartition = new Partition(sourcePartition);
  replicaPartition.setDbName(MAPPED_DB_NAME);
  replicaPartition.setTableName(MAPPED_TABLE_NAME);
  replicaPartition.setValues(Arrays.asList("A", "B"));

  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj();
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj();
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);

  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  columnStatisticsDesc
      .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues()));

  ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs);

  ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition,
      sourcePartitionStatistics);

  assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is(MAPPED_DB_NAME));
  assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is(MAPPED_TABLE_NAME));
  assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("one=A/two=B"));
  assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2));
  assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1));
  assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2));
}
 

static Partition newPartition(Table hiveTable, List<String> values, File location) {
  Partition partition = new Partition();
  partition.setDbName(hiveTable.getDbName());
  partition.setTableName(hiveTable.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveTable.getSd()));
  partition.getSd().setLocation(location.toURI().toString());
  return partition;
}
 

private Partition makeMetastoreBasePartitionObject(String dbName,
    String tblName, List<String> ptnVals, Table tbl) {
  Partition part4 = new Partition();
  part4.setDbName(dbName);
  part4.setTableName(tblName);
  part4.setValues(ptnVals);
  part4.setParameters(new HashMap<String, String>());
  part4.setSd(tbl.getSd().deepCopy());
  part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy());
  part4.setParameters(new HashMap<String, String>());
  return part4;
}
 

private static Partition newPartition(String databaseName, String tableName, String location) {
  Partition partition = new Partition();

  partition.setDbName(databaseName);
  partition.setTableName(tableName);
  partition.setParameters(new HashMap<String, String>());
  partition.setValues(Arrays.asList("01"));

  StorageDescriptor sd = new StorageDescriptor();
  sd.setLocation(location);
  partition.setSd(sd);

  return partition;
}
 

private Partition newPartition(
    String datbase,
    String table,
    StorageDescriptor tableStorageDescriptor,
    List<String> values,
    File location,
    String sourceTable,
    String sourceLocation,
    boolean addChecksum) {
  Partition partition = new Partition();
  partition.setDbName(datbase);
  partition.setTableName(table);
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(tableStorageDescriptor));
  partition.getSd().setLocation(location.toURI().toString());
  partition.setParameters(new HashMap<String, String>());
  if (sourceTable != null) {
    partition.getParameters().put(CircusTrainTableParameter.SOURCE_TABLE.parameterName(), sourceTable);
  }
  if (sourceLocation != null) {
    partition.getParameters().put(CircusTrainTableParameter.SOURCE_LOCATION.parameterName(), sourceLocation);
  }
  if (addChecksum) {
    partition.getParameters().put(CircusTrainTableParameter.PARTITION_CHECKSUM.parameterName(), location.getName());
  }
  return partition;
}
 

public static Partition newViewPartition(Table hiveView, List<String> values) {
  Partition partition = new Partition();
  partition.setDbName(hiveView.getDbName());
  partition.setTableName(hiveView.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveView.getSd()));
  partition.getSd().setLocation(null);
  return partition;
}
 

public static Partition newTablePartition(Table hiveTable, List<String> values, URI location) {
  Partition partition = new Partition();
  partition.setDbName(hiveTable.getDbName());
  partition.setTableName(hiveTable.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveTable.getSd()));
  partition.getSd().setLocation(location.toString());
  return partition;
}
 

public static Partition newPartition(Table table, String... partitionValues) {
  Partition partition = new Partition();
  partition.setTableName(table.getTableName());
  partition.setDbName(table.getDbName());
  partition.setValues(Arrays.asList(partitionValues));
  partition.setSd(table.getSd());
  partition.setParameters(new HashMap<String, String>());
  return partition;
}
 

@Test
public void newReplicaPartitionStatisticsWithTransformation() throws MetaException {
  sourceTable.setPartitionKeys(
      Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null)));

  Partition replicaPartition = new Partition(sourcePartition);
  replicaPartition.setDbName(MAPPED_DB_NAME);
  replicaPartition.setTableName(MAPPED_TABLE_NAME);
  replicaPartition.setValues(Arrays.asList("A", "B"));

  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj();
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj();
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);

  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  columnStatisticsDesc
      .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues()));

  ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs);

  ReplicaTableFactory factory = new ReplicaTableFactory(SOURCE_META_STORE_URIS, TableTransformation.IDENTITY,
      PartitionTransformation.IDENTITY, COLUMN_STATISTICS_TRANSFORMATION);

  ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition,
      sourcePartitionStatistics);

  assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is("new_db"));
  assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is("new_table"));
  assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("part=newPart"));
  assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2));
  assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1));
  assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2));
}
 

private Partition newHivePartition(
    String tableName,
    List<String> partitionValues,
    String location,
    Map<String, String> parameters) {
  Partition partition = new Partition();
  partition.setDbName(databaseName);
  partition.setTableName(tableName);
  partition.setValues(partitionValues);
  parameters.forEach((key, value) -> partition.putToParameters(key, value));
  partition.putToParameters(DATA_HIGHWAY_VERSION, DataHighwayVersion.VERSION);
  partition.putToParameters(DATA_HIGHWAY_LAST_REVISION, ISO_OFFSET_DATE_TIME.withZone(UTC).format(clock.instant()));
  partition.setSd(AvroStorageDescriptorFactory.create(location));
  return partition;
}
 

private Partition newPartition(String partitionValue, Path location1) {
  Partition partition = new Partition();
  partition.setValues(Lists.newArrayList(partitionValue));
  StorageDescriptor sd1 = new StorageDescriptor();
  sd1.setLocation(location1.toString());
  partition.setSd(sd1);
  Map<String, String> parameters = new HashMap<>();
  parameters.put(REPLICATION_EVENT.parameterName(), EVENT_ID);
  partition.setParameters(parameters);
  return partition;
}
 

private Partition newPartition(StorageDescriptor tableStorageDescriptor, List<String> values, File location) {
  Partition partition = new Partition();
  partition.setDbName(DATABASE);
  partition.setTableName(TABLE);
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(tableStorageDescriptor));
  partition.getSd().setLocation(location.toURI().toString());
  return partition;
}
 

@Test
public void dropPartition() throws Exception {
  HiveServer2Core server = setupServer();
  String tableName = "my_table";
  HiveMetaStoreClient client = server.getCore().newClient();

  try {
    Table table = createPartitionedTable(DATABASE, tableName, server);

    Partition partition = new Partition();
    partition.setDbName(DATABASE);
    partition.setTableName(tableName);
    partition.setValues(Arrays.asList("1"));
    partition.setSd(new StorageDescriptor(table.getSd()));
    partition.getSd().setLocation(
        String.format("file:%s/%s/%s/partcol=1", server.getCore().tempDir(), DATABASE, tableName));
    client.add_partition(partition);

    try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl());
        Statement statement = connection.createStatement()) {
      String dropPartitionHql = String.format("ALTER TABLE %s.%s DROP PARTITION (partcol=1)", DATABASE, tableName);
      statement.execute(dropPartitionHql);
    }

    List<Partition> partitions = client.listPartitions(DATABASE, tableName, (short) -1);
    assertThat(partitions.size(), is(0));
  } finally {
    client.close();
  }
  server.shutdown();
}
 

/**
 * Creates a Hive partition instance.
 */
public static Partition createHivePartition(String dbName, String tableName, List<String> values,
		StorageDescriptor sd, Map<String, String> parameters) {
	Partition partition = new Partition();
	partition.setDbName(dbName);
	partition.setTableName(tableName);
	partition.setValues(values);
	partition.setParameters(parameters);
	partition.setSd(sd);
	int currentTime = (int) (System.currentTimeMillis() / 1000);
	partition.setCreateTime(currentTime);
	partition.setLastAccessTime(currentTime);
	return partition;
}
 

private void createPartition(LinkedHashMap<String, String> partSpec, Path path) throws Exception {
	StorageDescriptor newSd = new StorageDescriptor(sd);
	newSd.setLocation(path.toString());
	Partition partition = HiveTableUtil.createHivePartition(database, tableName,
			new ArrayList<>(partSpec.values()), newSd, new HashMap<>());
	partition.setValues(new ArrayList<>(partSpec.values()));
	client.add_partition(partition);
}
 

/**
 * {@inheritDoc}
 */
@Override
public Partition metacatToHivePartition(final PartitionDto partitionDto, @Nullable final TableDto tableDto) {
    final Partition result = new Partition();

    final QualifiedName name = partitionDto.getName();
    List<String> values = Lists.newArrayListWithCapacity(16);
    String databaseName = null;
    String tableName = null;
    if (name != null) {
        if (name.getPartitionName() != null) {
            //
            // Unescape the partition name to get the right partition values.
            // Partition name always are escaped where as the parition values are not.
            //
            values = getPartValsFromName(tableDto, name.getPartitionName());
        }

        if (name.getDatabaseName() != null) {
            databaseName = name.getDatabaseName();
        }

        if (name.getTableName() != null) {
            tableName = name.getTableName();
        }
    }
    result.setValues(values);
    result.setDbName(databaseName);
    result.setTableName(tableName);

    Map<String, String> metadata = partitionDto.getMetadata();
    if (metadata == null) {
        metadata = Maps.newHashMap();
    }
    result.setParameters(metadata);

    result.setSd(fromStorageDto(partitionDto.getSerde(), tableName));
    final StorageDescriptor sd = result.getSd();
    if (tableDto != null) {
        if (sd.getSerdeInfo() != null && tableDto.getSerde() != null && Strings.isNullOrEmpty(
            sd.getSerdeInfo().getSerializationLib())) {
            sd.getSerdeInfo().setSerializationLib(tableDto.getSerde().getSerializationLib());
        }

        final List<FieldDto> fields = tableDto.getFields();
        if (fields == null) {
            sd.setCols(Collections.emptyList());
        } else {
            sd.setCols(fields.stream()
                .filter(field -> !field.isPartition_key())
                .map(this::metacatToHiveField)
                .collect(Collectors.toList()));
        }
    }

    final AuditDto auditDto = partitionDto.getAudit();
    if (auditDto != null) {
        if (auditDto.getCreatedDate() != null) {
            result.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate()));
        }
        if (auditDto.getLastModifiedDate() != null) {
            result.setLastAccessTime(dateToEpochSeconds(auditDto.getLastModifiedDate()));
        }
    }

    return result;
}