org.apache.hadoop.hive.metastore.MetaStoreUtils Java Examples

@Override
public void commitDropTable(Table tbl, boolean deleteData)
        throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));
    String tablename = getKuduTableName(tbl);
    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    try {
        if (deleteData && !isExternal) {
            client.deleteTable(tablename);
        }
    } catch (Exception ioe) {
        throw new MetaException("Error dropping table:" +tablename);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 

/**
 * Taken from HiveMetaStore#create_table_core
 * https://github.com/apache/hive/blob/rel/release-2.3.0/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java#L1370-L1383
 */
public static void validateTableObject(Table table, Configuration conf) throws InvalidObjectException {
  checkNotNull(table, "table cannot be null");
  checkNotNull(table.getSd(), "Table#StorageDescriptor cannot be null");

  if (!hiveShims.validateTableName(table.getTableName(), conf)) {
    throw new InvalidObjectException(table.getTableName() + " is not a valid object name");
  }
  String validate = MetaStoreUtils.validateTblColumns(table.getSd().getCols());
  if (validate != null) {
    throw new InvalidObjectException("Invalid column " + validate);
  }

  if (table.getPartitionKeys() != null) {
    validate = MetaStoreUtils.validateTblColumns(table.getPartitionKeys());
    if (validate != null) {
      throw new InvalidObjectException("Invalid partition column " + validate);
    }
  }
}
 

public SourceLocationManager getLocationManager(
    Table table,
    List<Partition> partitions,
    String eventId,
    Map<String, Object> copierOptions)
  throws IOException {
  if (MetaStoreUtils.isView(table)) {
    return new ViewLocationManager();
  }
  HdfsSnapshotLocationManager hdfsSnapshotLocationManager = new HdfsSnapshotLocationManager(getHiveConf(), eventId,
      table, partitions, snapshotsDisabled, sourceTableLocation, sourceCatalogListener);
  boolean ignoreMissingFolder = MapUtils.getBooleanValue(copierOptions,
      CopierOptions.IGNORE_MISSING_PARTITION_FOLDER_ERRORS, false);
  if (ignoreMissingFolder) {
    return new FilterMissingPartitionsLocationManager(hdfsSnapshotLocationManager, getHiveConf());
  }
  return hdfsSnapshotLocationManager;
}
 

@Override
public Table transform(Table table) {
  if (!MetaStoreUtils.isView(table)) {
    return table;
  }

  LOG.info("Translating HQL of view {}.{}", table.getDbName(), table.getTableName());
  String tableQualifiedName = Warehouse.getQualifiedName(table);
  String hql = hqlTranslator.translate(tableQualifiedName, table.getViewOriginalText());
  String expandedHql = hqlTranslator.translate(tableQualifiedName, table.getViewExpandedText());

  Table transformedView = new Table(table);
  transformedView.setViewOriginalText(hql);
  transformedView.setViewExpandedText(expandedHql);

  if (!replicaHiveConf.getBoolean(SKIP_TABLE_EXIST_CHECKS, false)) {
    LOG
        .info("Validating that tables used by the view {}.{} exist in the replica catalog", table.getDbName(),
            table.getTableName());
    validateReferencedTables(transformedView);
  }

  return transformedView;
}
 

/**
 * {@inheritDoc}
 */
@Override
public boolean partition_name_has_valid_characters(final List<String> partVals, final boolean throwException)
    throws TException {
    return requestWrapper("partition_name_has_valid_characters", new Object[]{partVals, throwException},
        () -> {
            Pattern pattern = null;
            final String partitionPattern = config.getHivePartitionWhitelistPattern();
            if (!Strings.isNullOrEmpty(partitionPattern)) {
                pattern = PATTERNS.getUnchecked(partitionPattern);
            }
            if (throwException) {
                MetaStoreUtils.validatePartitionNameCharacters(partVals, pattern);
                return true;
            } else {
                return MetaStoreUtils.partitionNameHasValidCharacters(partVals, pattern);
            }
        });
}
 

@Override
public void preCreateTable(Table table) throws MetaException {
  DynamoDBClient client = createDynamoDBClient(table);
  try {

    boolean isExternal = MetaStoreUtils.isExternalTable(table);

    if (!isExternal) {
      throw new MetaException("Only EXTERNAL tables are supported for DynamoDB.");
    }

    String tableName = HiveDynamoDBUtil.getDynamoDBTableName(table.getParameters()
        .get(DynamoDBConstants.TABLE_NAME), table.getTableName());
    TableDescription tableDescription = client.describeTable(tableName);

    checkTableStatus(tableDescription);
    checkTableSchemaMapping(tableDescription, table);
    checkTableSchemaType(tableDescription, table);
  } finally {
    client.close();
  }
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath, "Table path cannot be null");
	checkNotNull(partitionSpec, "CatalogPartitionSpec cannot be null");

	Table hiveTable = getHiveTable(tablePath);

	ensurePartitionedTable(tablePath, hiveTable);

	try {
		// partition spec can be partial
		List<String> partialVals = MetaStoreUtils.getPvals(hiveTable.getPartitionKeys(), partitionSpec.getPartitionSpec());
		return client.listPartitionNames(tablePath.getDatabaseName(), tablePath.getObjectName(), partialVals,
			(short) -1).stream().map(HiveCatalog::createPartitionSpec).collect(Collectors.toList());
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list partitions of table %s", tablePath), e);
	}
}
 

@Override
public void rollbackCreateTable(Table tbl) throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));
    String tablename = getKuduTableName(tbl);
    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    try {
        if ( client.tableExists(tablename) && !isExternal) {
            client.deleteTable(tablename);
        }
    } catch (Exception ioe) {
        throw new MetaException("Error dropping table while rollback of create table:" +tablename);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 

/**
 * First tries getting the {@code FieldSchema}s from the {@code HiveRegistrationUnit}'s columns, if set.
 * Else, gets the {@code FieldSchema}s from the deserializer.
 */
private static List<FieldSchema> getFieldSchemas(HiveRegistrationUnit unit) {
  List<Column> columns = unit.getColumns();
  List<FieldSchema> fieldSchemas = new ArrayList<>();
  if (columns != null && columns.size() > 0) {
    fieldSchemas = getFieldSchemas(columns);
  } else {
    Deserializer deserializer = getDeserializer(unit);
    if (deserializer != null) {
      try {
        fieldSchemas = MetaStoreUtils.getFieldsFromDeserializer(unit.getTableName(), deserializer);
      } catch (SerDeException | MetaException e) {
        LOG.warn("Encountered exception while getting fields from deserializer.", e);
      }
    }
  }
  return fieldSchemas;
}
 

@Override
public void commitDropTable(Table table, boolean deleteData) throws MetaException {
  //TODO: Should this be implemented to drop the table and its data from cassandra
  boolean isExternal = MetaStoreUtils.isExternalTable(table);
  if (deleteData && !isExternal) {
    CassandraManager manager = new CassandraManager(table);

    try {
      //open connection to cassandra
      manager.openConnection();
      //drop the table
      manager.dropTable();
    } finally {
      manager.closeConnection();
    }
  }
}
 

@Test
public void newTable() {
  TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME,
      REPLICA_DATA_DESTINATION, FULL);
  Table replica = replicaAndStats.getTable();

  assertThat(replica.getDbName(), is(sourceTable.getDbName()));
  assertThat(replica.getTableName(), is(sourceTable.getTableName()));
  assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT));
  assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT));
  assertThat(replica.getSd().getLocation(), is(REPLICA_DATA_DESTINATION.toUri().toString()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(TABLE_LOCATION));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
  assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
  assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
  assertThat(replica.getTableType(), is(TableType.EXTERNAL_TABLE.name()));
  assertThat(replica.getParameters().get("EXTERNAL"), is("TRUE"));
  assertTrue(MetaStoreUtils.isExternalTable(replica));

  assertThat(replicaAndStats.getStatistics(), is(nullValue()));
}
 

@Override
public void preCreateTable(Table table) throws MetaException {
  boolean isExternal = MetaStoreUtils.isExternalTable(table);

  if (!isExternal) {
    throw new MetaException("Cassandra tables must be external.");
  }

  if (table.getSd().getLocation() != null) {
    throw new MetaException("LOCATION may not be specified for Cassandra.");
  }

  CassandraManager manager = new CassandraManager(table);

  try {
    //open connection to cassandra
    manager.openConnection();
    KsDef ks = manager.getKeyspaceDesc();

    //create the column family if it doesn't exist.
    manager.createCFIfNotFound(ks);
  } catch(NotFoundException e) {
    manager.createKeyspaceWithColumns();
  } finally {
    manager.closeConnection();
  }
}
 

@Override
public void preCreateTable(Table tbl) throws MetaException {
    if(!MetaStoreUtils.isExternalTable(tbl)) {
        throw new MetaException("Table must be external.");
    }
    // TODO Auto-generated method stub
}
 

private boolean startAddPartition(
        final RawStore ms, final Partition part, final boolean ifNotExists) throws MetaException, TException {
    MetaStoreUtils.validatePartitionNameCharacters(part.getValues(),
            partitionValidationPattern);
    final boolean doesExist = ms.doesPartitionExist(
            part.getDbName(), part.getTableName(), part.getValues());
    if (doesExist && !ifNotExists) {
        throw new AlreadyExistsException("Partition already exists: " + part);
    }
    return !doesExist;
}
 

@Override
public void preCreateTable(Table tbl) throws MetaException {

    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    if (isExternal) {
        Log.info("Creating External table for Splice...");
    }

    String inputTableName = tbl.getParameters().get(MRConstants.SPLICE_TABLE_NAME);
    if (inputTableName == null)
        throw new MetaException("Wrong param, you are missing " +
        		MRConstants.SPLICE_TABLE_NAME + " ? ");

    // We can choose to support user define column mapping.
    // But currently I don't think it is necessary
    // We map all columns from Splice Table to Hive Table.
    String connStr = tbl.getParameters().get(MRConstants.SPLICE_JDBC_STR);
    if (connStr == null)
        throw new MetaException("Wrong param, did you mean " +
        		MRConstants.SPLICE_JDBC_STR + " ? ");
    if (sqlUtil == null)
        sqlUtil = SMSQLUtil.getInstance(connStr);
    if (inputTableName != null) {
        inputTableName = inputTableName.trim();
        checkTableExists(inputTableName);
    }
}
 

private HiveTestDataGenerator(final String dbDir, final String whDir) throws Exception {
  this.dbDir = dbDir;
  this.whDir = whDir;

  final HiveConf conf = new HiveConf();

  HiveConf.setBoolVar(conf, HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, false);
  HiveConf.setBoolVar(conf, HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL, true);
  HiveConf.setVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE, whDir);
  HiveConf.setVar(conf, HiveConf.ConfVars.METASTORECONNECTURLKEY, String.format("jdbc:derby:;databaseName=%s;create=true", dbDir));
  port = MetaStoreUtils.startMetaStore(conf);

  config = Maps.newHashMap();
  config.put(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
}
 

/**
 * Wrapper around {@link MetaStoreUtils#getPartitionMetadata(Partition, Table)} which also adds parameters from table
 * to properties returned by {@link MetaStoreUtils#getPartitionMetadata(Partition, Table)}.
 *
 * @param partition the source of partition level parameters
 * @param table     the source of table level parameters
 * @return properties
 */
public static Properties buildPartitionProperties(final Partition partition, final Table table) {
  final Properties properties = MetaStoreUtils.getPartitionMetadata(partition, table);

  // SerDe expects properties from Table, but above call doesn't add Table properties.
  // Include Table properties in final list in order to not to break SerDes that depend on
  // Table properties. For example AvroSerDe gets the schema from properties (passed as second argument)
  for (Map.Entry<String, String> entry : table.getParameters().entrySet()) {
    if (entry.getKey() != null && entry.getValue() != null) {
      properties.put(entry.getKey(), entry.getValue());
    }
  }

  return properties;
}
 

public static void assertTableIsExternal(
    HiveMetaStoreClient client, String db, String name)
    throws MetaException, TException {
  final Table table = client.getTable(db, name);
  Assert.assertTrue("Table should be external db:" + db + " table:" + table,
      MetaStoreUtils.isExternalTable(table) &&
      TableType.EXTERNAL_TABLE.toString().equals(table.getTableType()));
}
 

public static void assertTableIsManaged(
    HiveMetaStoreClient client, String db, String name)
    throws MetaException, TException {
  final Table table = client.getTable(db, name);
  Assert.assertTrue("Table should be external db:" + db + " table:" + table,
      !MetaStoreUtils.isExternalTable(table) &&
      !MetaStoreUtils.isIndexTable(table) &&
      TableType.MANAGED_TABLE.toString().equals(table.getTableType()));
}
 

public SourceLocationManager getLocationManager(Table table, String eventId) throws IOException {
  if (MetaStoreUtils.isView(table)) {
    return new ViewLocationManager();
  }
  return new HdfsSnapshotLocationManager(getHiveConf(), eventId, table, snapshotsDisabled, sourceTableLocation,
      sourceCatalogListener);
}
 

@Override
public void validatePartitionNameCharacters(List<String> part_vals) throws TException, MetaException {
    try {
        String partitionValidationRegex = conf.getVar(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN);
        Pattern partitionValidationPattern = Strings.isNullOrEmpty(partitionValidationRegex) ? null
              : Pattern.compile(partitionValidationRegex);
        MetaStoreUtils.validatePartitionNameCharacters(part_vals, partitionValidationPattern);
    } catch (Exception e){
        if (e instanceof MetaException) {
            throw (MetaException) e;
        } else {
            throw new MetaException(e.getMessage());
        }
    }
}
 

@Override
public boolean requireCalStats(
    Configuration conf,
    Partition oldPart,
    Partition newPart,
    Table tbl,
    EnvironmentContext environmentContext) {
  return MetaStoreUtils.requireCalStats(conf, oldPart, newPart, tbl, environmentContext);
}
 

@Override
public boolean updateTableStatsFast(
    Database db,
    Table tbl,
    Warehouse wh,
    boolean madeDir,
    boolean forceRecompute,
    EnvironmentContext environmentContext
) throws MetaException {
  return MetaStoreUtils.updateTableStatsFast(db, tbl, wh, madeDir, forceRecompute, environmentContext);
}
 

@Override
public boolean requireCalStats(
    Configuration conf,
    Partition oldPart,
    Partition newPart,
    Table tbl,
    EnvironmentContext environmentContext) {
  return MetaStoreUtils.requireCalStats(conf, oldPart, newPart, tbl);
}
 

@Override
public boolean updateTableStatsFast(
    Database db,
    Table tbl,
    Warehouse wh,
    boolean madeDir,
    boolean forceRecompute,
    EnvironmentContext environmentContext
) throws MetaException {
  return MetaStoreUtils.updateUnpartitionedTableStatsFast(db, tbl, wh, madeDir, forceRecompute);
}
 

@Override
public void validatePartitionNameCharacters(List<String> part_vals) throws TException, MetaException {
  try {
    String partitionValidationRegex = conf.getVar(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN);
    Pattern partitionValidationPattern = Strings.isNullOrEmpty(partitionValidationRegex) ? null
          : Pattern.compile(partitionValidationRegex);
    MetaStoreUtils.validatePartitionNameCharacters(part_vals, partitionValidationPattern);
  } catch (Exception e){
    if (e instanceof MetaException) {
      throw (MetaException) e;
    } else {
      throw new MetaException(e.getMessage());
    }
  }
}
 

/**
 * Extracts the db_name(s) and table_name(s) corresponding to the given pattern.
 * pattern is the Hive table name or pattern that the user enters in the CREATE EXTERNAL TABLE statement
 * or when querying HCatalog table.
 * It can be either <code>table_name_pattern</code> or <code>db_name_pattern.table_name_pattern</code>.
 *
 * @param client  MetaStoreClient client
 * @param pattern Hive table name or pattern
 * @return list of {@link Metadata.Item} objects holding the full table name
 */
public List<Metadata.Item> extractTablesFromPattern(IMetaStoreClient client, String pattern) {

    String dbPattern, tablePattern;
    String errorMsg = " is not a valid Hive table name. "
            + "Should be either <table_name> or <db_name.table_name>";

    if (StringUtils.isBlank(pattern)) {
        throw new IllegalArgumentException("empty string" + errorMsg);
    }

    String[] rawTokens = pattern.split("[.]");
    ArrayList<String> tokens = new ArrayList<>();
    for (String tok : rawTokens) {
        if (StringUtils.isBlank(tok)) {
            continue;
        }
        tokens.add(tok.trim());
    }

    if (tokens.size() == 1) {
        dbPattern = MetaStoreUtils.DEFAULT_DATABASE_NAME;
        tablePattern = tokens.get(0);
    } else if (tokens.size() == 2) {
        dbPattern = tokens.get(0);
        tablePattern = tokens.get(1);
    } else {
        throw new IllegalArgumentException("\"" + pattern + "\"" + errorMsg);
    }

    return getTablesFromPattern(client, dbPattern, tablePattern);
}
 

private void validate(TableReplication tableReplication, Source source, Replica replica) {
  source.getDatabase(tableReplication.getSourceTable().getDatabaseName());
  replica.getDatabase(tableReplication.getReplicaDatabaseName());

  TableAndStatistics sourceTableAndStatistics = source.getTableAndStatistics(tableReplication);
  if (tableReplication.getReplicationMode() != ReplicationMode.METADATA_MIRROR
      && MetaStoreUtils.isView(sourceTableAndStatistics.getTable())) {
    throw new CircusTrainException(String
        .format("Cannot replicate view %s. Only %s is supported for views",
            tableReplication.getSourceTable().getQualifiedName(), ReplicationMode.METADATA_MIRROR.name()));
  }
}
 

@Test
public void newView() {
  sourceTableAndStats.getTable().setTableType(TableType.VIRTUAL_VIEW.name());
  sourceTableAndStats.getTable().getSd().setInputFormat(null);
  sourceTableAndStats.getTable().getSd().setOutputFormat(null);
  sourceTableAndStats.getTable().getSd().setLocation(null);

  TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME,
      null, FULL);
  Table replica = replicaAndStats.getTable();

  assertThat(replica.getDbName(), is(sourceTable.getDbName()));
  assertThat(replica.getTableName(), is(sourceTable.getTableName()));
  assertThat(replica.getSd().getInputFormat(), is(nullValue()));
  assertThat(replica.getSd().getOutputFormat(), is(nullValue()));
  assertThat(replica.getSd().getLocation(), is(nullValue()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(""));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
  assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
  assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
  assertThat(replica.getTableType(), is(TableType.VIRTUAL_VIEW.name()));
  assertTrue(MetaStoreUtils.isView(replica));

  assertThat(replicaAndStats.getStatistics(), is(nullValue()));
}
 

@Override
public void preCreateTable(Table tbl)
        throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));

    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);

    if (isExternal) {
        //TODO: Check if Kudu table exists to allow external table.
        //TODO: Check if column and types are compatible with existing Kudu table.
        throw new MetaException("External Table to Kudu not yet supported.");
    }
    if (tbl.getSd().getLocation() != null) {
        throw new MetaException("LOCATION may not be specified for Kudu");
    }

    String tablename = getKuduTableName(tbl);

    try {
        List<String> keyColumns = Arrays.asList(tbl.getParameters().get(HiveKuduConstants.KEY_COLUMNS).split("\\s*,\\s*"));

        List<FieldSchema> tabColumns = tbl.getSd().getCols();

        int numberOfCols = tabColumns.size();
        List<ColumnSchema> columns = new ArrayList<>(numberOfCols);

        for (FieldSchema fields : tabColumns) {

            ColumnSchema columnSchema = new ColumnSchema
                    .ColumnSchemaBuilder(fields.getName(), HiveKuduBridgeUtils.hiveTypeToKuduType(fields.getType()))
                    .key(keyColumns.contains(fields.getName()))
                    .nullable(!keyColumns.contains(fields.getName()))
                    .build();

            columns.add(columnSchema);
        }

        Schema schema = new Schema(columns);

        printSchema(schema);

        CreateTableOptions createTableOptions = new CreateTableOptions();

        //TODO : add support for partition and buckets
        client.createTable(tablename, schema, createTableOptions);

    } catch (Exception se) {
        throw new MetaException("Error creating Kudu table: " + tablename + ":" + se);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}