Java Code Examples for org.apache.hadoop.hive.metastore.Warehouse#makePartName()

The following examples show how to use org.apache.hadoop.hive.metastore.Warehouse#makePartName() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: GlueMetastoreClientDelegate.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up 
/**
 * Taken from HiveMetaStore#append_partition_common
 */
private org.apache.hadoop.hive.metastore.api.Partition buildPartitionFromValues(
  org.apache.hadoop.hive.metastore.api.Table table, List<String> values) throws MetaException {
  org.apache.hadoop.hive.metastore.api.Partition partition = new org.apache.hadoop.hive.metastore.api.Partition();
  partition.setDbName(table.getDbName());
  partition.setTableName(table.getTableName());
  partition.setValues(values);
  partition.setSd(table.getSd().deepCopy());

  Path partLocation = new Path(table.getSd().getLocation(), Warehouse.makePartName(table.getPartitionKeys(), values));
  partition.getSd().setLocation(partLocation.toString());

  long timeInSecond = System.currentTimeMillis() / MILLISECOND_TO_SECOND_FACTOR;
  partition.setCreateTime((int) timeInSecond);
  partition.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(timeInSecond));
  return partition;
}
Example 2
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up 
@Override
public org.apache.hadoop.hive.metastore.api.Partition getPartitionWithAuthInfo(
      String databaseName, String tableName, List<String> values,
      String userName, List<String> groupNames)
      throws MetaException, UnknownTableException, NoSuchObjectException, TException {

    // TODO move this into the service
    org.apache.hadoop.hive.metastore.api.Partition partition = getPartition(databaseName, tableName, values);
    org.apache.hadoop.hive.metastore.api.Table table = getTable(databaseName, tableName);
    if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
        String partName = Warehouse.makePartName(table.getPartitionKeys(), values);
        HiveObjectRef obj = new HiveObjectRef();
        obj.setObjectType(HiveObjectType.PARTITION);
        obj.setDbName(databaseName);
        obj.setObjectName(tableName);
        obj.setPartValues(values);
        org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet privilegeSet =
              this.get_privilege_set(obj, userName, groupNames);
        partition.setPrivileges(privilegeSet);
    }

    return partition;
}
Example 3
Source File: AWSCatalogMetastoreClientTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAppendPartitionByName() throws Exception {    
  List<String> values = Arrays.asList("foo");
  when(glueClient.getTable(any(GetTableRequest.class)))
      .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  Path partLocation = new Path(testTable.getSd().getLocation(), Warehouse
          .makePartName(testTable.getPartitionKeys(), values));
  setupMockWarehouseForPath(partLocation, false, true);
  mockBatchCreatePartitionsSucceed();

  org.apache.hadoop.hive.metastore.api.Partition res = metastoreClient.appendPartition(
      testDB.getName(),
      testTable.getTableName(),
      testTable.getPartitionKeys().get(0).getName() + "=foo");
  assertThat(res.getValues(), is(values));
  assertDaemonThreadPools();
}
Example 4
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up 
@Override
public org.apache.hadoop.hive.metastore.api.Partition getPartitionWithAuthInfo(
      String databaseName, String tableName, List<String> values,
      String userName, List<String> groupNames)
      throws MetaException, UnknownTableException, NoSuchObjectException, TException {

  // TODO move this into the service
  org.apache.hadoop.hive.metastore.api.Partition partition = getPartition(databaseName, tableName, values);
  org.apache.hadoop.hive.metastore.api.Table table = getTable(databaseName, tableName);
  if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
    String partName = Warehouse.makePartName(table.getPartitionKeys(), values);
    HiveObjectRef obj = new HiveObjectRef();
    obj.setObjectType(HiveObjectType.PARTITION);
    obj.setDbName(databaseName);
    obj.setObjectName(tableName);
    obj.setPartValues(values);
    org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet privilegeSet =
          this.get_privilege_set(obj, userName, groupNames);
    partition.setPrivileges(privilegeSet);
  }

  return partition;
}
Example 5
Source File: GlueMetastoreClientDelegateTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAppendPartition() throws Exception {
  List<String> values = ImmutableList.of("foo");
  when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable(testTbl));
  Path partLocation = new Path(testTbl.getStorageDescriptor().getLocation(),
    Warehouse.makePartName(CatalogToHiveConverter.convertFieldSchemaList(testTbl.getPartitionKeys()), values));
  setupMockWarehouseForPath(partLocation, false, true);
  mockBatchCreatePartitionsSucceed();

  org.apache.hadoop.hive.metastore.api.Partition res =
    metastoreClientDelegate.appendPartition(testDb.getName(), testTbl.getName(), values);

  verify(wh, times(1)).mkdirs(partLocation, true);
  assertThat(res.getValues(), is(values));
}
Example 6
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up 
private Path getDestinationPathForRename(String dbName, org.apache.hadoop.hive.metastore.api.Table tbl, org.apache.hadoop.hive.metastore.api.Partition newPartition)
      throws InvalidOperationException, MetaException, TException {
    try {
        Path destPath = new Path(hiveShims.getDefaultTablePath(getDatabase(dbName), tbl.getTableName(), wh),
              Warehouse.makePartName(tbl.getPartitionKeys(), newPartition.getValues()));
        return constructRenamedPath(destPath, new Path(newPartition.getSd().getLocation()));
    } catch (NoSuchObjectException e) {
        throw new InvalidOperationException(
              "Unable to change partition or table. Database " + dbName + " does not exist"
                    + " Check metastore logs for detailed stack." + e.getMessage());
    }
}
Example 7
Source File: AWSCatalogMetastoreClientTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRenamePartitionForHiveManagedTable() throws Exception {
  String dbName = testDB.getName();
  String tblName = testTable.getTableName();
  List<String> partitionValues = testPartition.getValues();
  Partition catalogPartition = HiveToCatalogConverter.convertPartition(testPartition);
  
  Partition newPartition = new Partition()
                                .withDatabaseName(dbName).withTableName(tblName)
                                .withValues(Lists.newArrayList("newval"))
                                .withStorageDescriptor(catalogPartition.getStorageDescriptor());
  
  Path srcPath = new Path(testPartition.getSd().getLocation());
  Path expectedDestPath = new Path("/db/tbl/" ,
      Warehouse.makePartName(testTable.getPartitionKeys(), newPartition.getValues()));
      
  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(catalogPartition));

  FileSystem fs = mock(FileSystem.class);
  when(fs.getUri()).thenReturn(new URI("s3://bucket"));
  when(hiveShims.getDefaultTablePath(any(org.apache.hadoop.hive.metastore.api.Database.class), anyString(), wh))
      .thenReturn(new Path("/db/tbl"));
  when(wh.getFs(any(Path.class))).thenReturn(fs);
  when(fs.exists(srcPath)).thenReturn(true);
  when(fs.exists(expectedDestPath)).thenReturn(false);
  when(wh.mkdirs(expectedDestPath.getParent(), true)).thenReturn(true);       
  
  metastoreClient.renamePartition(dbName, tblName, partitionValues,
      CatalogToHiveConverter.convertPartition(newPartition));
  
  // Verify catalog service is called , dest path's parent dirs are created and dfs rename is done
  verify(glueClient, times(1)).updatePartition(any(UpdatePartitionRequest.class));
  verify(wh).mkdirs(expectedDestPath.getParent(), true);
  verify(wh).renameDir(srcPath, expectedDestPath, true);
}
Example 8
Source File: AWSCatalogMetastoreClientTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up 
@Test(expected=InvalidOperationException.class)
public void testRenamePartitionDestinationAlreadyExists() throws Exception {
  String dbName = testDB.getName();
  String tblName = testTable.getTableName();
  List<String> partitionValues = testPartition.getValues();
  Partition catalogPartition = HiveToCatalogConverter.convertPartition(testPartition);
  
  Partition newPartition = new Partition()
                                .withDatabaseName(dbName).withTableName(tblName)
                                .withValues(Lists.newArrayList("newval"))
                                .withStorageDescriptor(catalogPartition.getStorageDescriptor());
  
  Path srcPath = new Path(testPartition.getSd().getLocation());
  Path expectedDestPath = new Path("/db/tbl/" , Warehouse.makePartName(testTable.getPartitionKeys(),
      newPartition.getValues()));
      
  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(catalogPartition));
  
  
  FileSystem fs = mock(FileSystem.class);
  when(fs.getUri()).thenReturn(new URI("s3://bucket"));
  when(hiveShims.getDefaultTablePath(any(org.apache.hadoop.hive.metastore.api.Database.class), anyString(), wh))
      .thenReturn(new Path("/db/tbl"));
  when(wh.getFs(any(Path.class))).thenReturn(fs);
  when(fs.exists(srcPath)).thenReturn(true);
  when(fs.exists(expectedDestPath)).thenReturn(true);
      
  metastoreClient.renamePartition(dbName, tblName, partitionValues,
      CatalogToHiveConverter.convertPartition(newPartition));
}
Example 9
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up 
private Path getDestinationPathForRename(String dbName, org.apache.hadoop.hive.metastore.api.Table tbl, org.apache.hadoop.hive.metastore.api.Partition newPartition)
      throws InvalidOperationException, MetaException, TException {
  try {
    Path destPath = new Path(hiveShims.getDefaultTablePath(getDatabase(dbName), tbl.getTableName(), wh),
          Warehouse.makePartName(tbl.getPartitionKeys(), newPartition.getValues()));
    return constructRenamedPath(destPath, new Path(newPartition.getSd().getLocation()));
  } catch (NoSuchObjectException e) {
    throw new InvalidOperationException(
          "Unable to change partition or table. Database " + dbName + " does not exist"
                + " Check metastore logs for detailed stack." + e.getMessage());
  }
}
Example 10
Source File: PartitionsAndStatistics.java    From circus-train with Apache License 2.0 5 votes vote down vote up 
private static String getPartitionName(List<FieldSchema> partitionKeys, Partition partition) {
  try {
    return Warehouse.makePartName(partitionKeys, partition.getValues());
  } catch (MetaException e) {
    throw new RuntimeException(e);
  }
}
Example 11
Source File: DestructiveReplica.java    From circus-train with Apache License 2.0 5 votes vote down vote up 
private void dropAndDeletePartitions(CloseableMetaStoreClient client, Predicate<String> shouldDelete)
  throws MetaException, TException, NoSuchObjectException {
  Table replicaTable = client.getTable(databaseName, tableName);
  List<FieldSchema> partitionKeys = replicaTable.getPartitionKeys();
  if (partitionKeys == null || partitionKeys.isEmpty()) {
    // unpartitioned table nothing to delete
    return;
  }
  PartitionIterator partitionIterator = new PartitionIterator(client, replicaTable, (short) 1000);
  while (partitionIterator.hasNext()) {
    Partition replicaPartition = partitionIterator.next();
    List<String> values = replicaPartition.getValues();
    String partitionName = Warehouse.makePartName(partitionKeys, values);
    if (shouldDelete.apply(partitionName)) {
      log
          .info("Dropping partition for replica table: "
              + databaseName
              + "."
              + tableName
              + ", partition value: '"
              + partitionName
              + "'");
      client.dropPartition(databaseName, tableName, partitionName, DELETE_DATA);
      Path oldLocation = locationAsPath(replicaPartition);
      String oldEventId = replicaPartition.getParameters().get(REPLICATION_EVENT.parameterName());
      cleanupLocationManager.addCleanupLocation(oldEventId, oldLocation);
    }
  }
}
Example 12
Source File: ReplicaTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up 
private String partitionName(String... values) {
  try {
    return Warehouse.makePartName(PARTITIONS, Arrays.asList(values));
  } catch (MetaException e) {
    throw new RuntimeException(e);
  }
}
Example 13
Source File: HiveDifferences.java    From circus-train with Apache License 2.0 5 votes vote down vote up 
private static String partitionName(Table table, Partition partition) {
  try {
    return Warehouse.makePartName(table.getPartitionKeys(), partition.getValues());
  } catch (MetaException e) {
    throw new CircusTrainException("Unable to build partition name for partition values "
        + partition.getValues()
        + " of table "
        + Warehouse.getQualifiedName(table), e);
  }
}
Example 14
Source File: PartitionUtil.java    From metacat with Apache License 2.0 5 votes vote down vote up 
/**
 * Generate partition name from the <code>partValues</code>.
 *
 * @param partitionKeys list of partition keys
 * @param partValues list of partition values
 * @return partition name
 */
public static String makePartName(final List<FieldSchema> partitionKeys, final List<String> partValues) {
    try {
        return Warehouse.makePartName(partitionKeys, partValues);
    } catch (MetaException e) {
        throw new InvalidMetaException("Failed making the part name from the partition values", e);
    }
}
Example 15
Source File: HiveConnectorPartitionService.java    From metacat with Apache License 2.0 5 votes vote down vote up 
private String getNameOfPartition(final Table table, final Partition partition) {
    try {
        return Warehouse.makePartName(table.getPartitionKeys(), partition.getValues());
    } catch (TException e) {
        throw new InvalidMetaException("One or more partition names are invalid.", e);
    }
}
Example 16
Source File: AWSCatalogMetastoreClientTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRenamePartitionRevertWhenMetastoreoperationFails() throws Exception {
  String dbName = testDB.getName();
  String tblName = testTable.getTableName();
  List<String> partitionValues = testPartition.getValues();
  Partition catalogPartition = HiveToCatalogConverter.convertPartition(testPartition);
  
  Partition newPartition = new Partition()
                              .withDatabaseName(dbName).withTableName(tblName)
                              .withValues(Lists.newArrayList("newval"))
                              .withStorageDescriptor(catalogPartition.getStorageDescriptor());
  
  Path srcPath = new Path(testPartition.getSd().getLocation());
  Path expectedDestPath = new Path("/db/tbl/" ,
      Warehouse.makePartName(testTable.getPartitionKeys(), newPartition.getValues()));
      
  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(catalogPartition));

  FileSystem fs = mock(FileSystem.class);
  when(fs.getUri()).thenReturn(new URI("s3://bucket"));
  when(hiveShims.getDefaultTablePath(any(org.apache.hadoop.hive.metastore.api.Database.class), anyString(), wh))
      .thenReturn(new Path("/db/tbl"));
  when(wh.getFs(any(Path.class))).thenReturn(fs);
  when(fs.exists(srcPath)).thenReturn(true);
  when(fs.exists(expectedDestPath)).thenReturn(false);
  
  // Fail directory creation
  when(wh.mkdirs(expectedDestPath.getParent(), true)).thenReturn(false);
  
  boolean exceptionThrown = false;
  try {
    metastoreClient.renamePartition(dbName, tblName, partitionValues,
        CatalogToHiveConverter.convertPartition(newPartition));
  } catch(InvalidOperationException e) {
    exceptionThrown = true;
  }
  assertTrue(exceptionThrown);
  
  // Verify catalog service is called , dest path's parent dirs are created and dfs rename is done
  verify(glueClient, times(2)).updatePartition(any(UpdatePartitionRequest.class));
}