org.apache.hadoop.hive.metastore.TableType Java Examples

@Override
public Table newHiveTable(
    String databaseName,
    String tableName,
    String partitionColumnName,
    String location,
    Schema schema,
    int version) {

  Table table = new Table();
  table.setDbName(databaseName);
  table.setTableName(tableName);

  table.setTableType(TableType.EXTERNAL_TABLE.toString());
  table.putToParameters("EXTERNAL", "TRUE");
  addRoadAnnotations(table);

  URI schemaUri = uriResolver.resolve(schema, table.getTableName(), version);
  table.putToParameters(AVRO_SCHEMA_URL, schemaUri.toString());
  table.putToParameters(AVRO_SCHEMA_VERSION, Integer.toString(version));
  table.setPartitionKeys(Arrays.asList(new FieldSchema(partitionColumnName, "string", null)));

  table.setSd(AvroStorageDescriptorFactory.create(location));

  return table;
}
 

/**
 * @return boolean
 *    true  -> directory created
 *    false -> directory not created
 */
public boolean validateNewTableAndCreateDirectory(org.apache.hadoop.hive.metastore.api.Table tbl) throws TException {
  checkNotNull(tbl, "tbl cannot be null");
  if (tableExists(tbl.getDbName(), tbl.getTableName())) {
    throw new AlreadyExistsException("Table " + tbl.getTableName() + " already exists.");
  }
  validateTableObject(tbl, conf);

  if (TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) {
    // we don't need to create directory for virtual views
    return false;
  }

  if (StringUtils.isEmpty(tbl.getSd().getLocation())) {
    org.apache.hadoop.hive.metastore.api.Database db = getDatabase(tbl.getDbName());
    tbl.getSd().setLocation(hiveShims.getDefaultTablePath(db, tbl.getTableName(), wh).toString());
  } else {
    tbl.getSd().setLocation(wh.getDnsPath(new Path(tbl.getSd().getLocation())).toString());
  }

  Path tblPath = new Path(tbl.getSd().getLocation());
  return makeDirs(wh, tblPath);
}
 

@Override
public List<String> getViews(IMetaStoreClient client, String databaseName) throws UnknownDBException, TException {
	try {
		Method method = client.getClass().getMethod("getTables", String.class, String.class, TableType.class);
		return (List<String>) method.invoke(client, databaseName, null, TableType.VIRTUAL_VIEW);
	} catch (InvocationTargetException ite) {
		Throwable targetEx = ite.getTargetException();
		if (targetEx instanceof TException) {
			throw (TException) targetEx;
		} else {
			throw new CatalogException(String.format("Failed to get views for %s", databaseName), targetEx);
		}
	} catch (NoSuchMethodException | IllegalAccessException e) {
		throw new CatalogException(String.format("Failed to get views for %s", databaseName), e);
	}
}
 

@Test
public void testImportThatUpdatesRegisteredTable() throws Exception {
    setupDB(hiveClient, TEST_DB_NAME);

    List<Table> hiveTables = setupTables(hiveClient, TEST_DB_NAME, TEST_TABLE_NAME);

    returnExistingDatabase(TEST_DB_NAME, atlasClient, CLUSTER_NAME);

    // return existing table
    when(atlasClient.getEntity(HiveDataTypes.HIVE_TABLE.getName(),
        AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, TEST_DB_NAME, TEST_TABLE_NAME)))
        .thenReturn(getEntityReference(HiveDataTypes.HIVE_TABLE.getName(), "82e06b34-9151-4023-aa9d-b82103a50e77"));
    when(atlasClient.getEntity("82e06b34-9151-4023-aa9d-b82103a50e77")).thenReturn(createTableReference());
    String processQualifiedName = HiveMetaStoreBridge.getTableProcessQualifiedName(CLUSTER_NAME, hiveTables.get(0));
    when(atlasClient.getEntity(HiveDataTypes.HIVE_PROCESS.getName(),
        AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, processQualifiedName)).thenReturn(getEntityReference(HiveDataTypes.HIVE_PROCESS.getName(), "82e06b34-9151-4023-aa9d-b82103a50e77"));

    HiveMetaStoreBridge bridge = new HiveMetaStoreBridge(CLUSTER_NAME, hiveClient, atlasClient);
    bridge.importHiveMetadata(true);

    // verify update is called on table
    verify(atlasClient).updateEntity(eq("82e06b34-9151-4023-aa9d-b82103a50e77"),
            (Referenceable) argThat(new MatchesReferenceableProperty(HiveMetaStoreBridge.TABLE_TYPE_ATTR,
                    TableType.EXTERNAL_TABLE.name())));
}
 

@Test
public void testValidateTableAndCreateDirectoryVirtualView() throws Exception {
  testTbl.setTableType(TableType.VIRTUAL_VIEW.toString());
  testTbl.getStorageDescriptor().setLocation(null);
  org.apache.hadoop.hive.metastore.api.Table hiveTbl = CatalogToHiveConverter.convertTable(testTbl, testTbl.getDatabaseName());

  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(testDb));
  when(glueClient.getTable(new GetTableRequest()
    .withDatabaseName(testTbl.getDatabaseName()).withName(testTbl.getName())))
    .thenThrow(EntityNotFoundException.class);

  assertFalse(metastoreClientDelegate.validateNewTableAndCreateDirectory(hiveTbl));
  assertNull(testTbl.getStorageDescriptor().getLocation());
  verify(wh, never()).mkdirs(any(Path.class), anyBoolean());
}
 

private Table constructParquetTable(String database, String tableName, Schema schema, Partitioner partitioner) throws HiveMetaStoreException {
  Table table = new Table(database, tableName);
  table.setTableType(TableType.EXTERNAL_TABLE);
  table.getParameters().put("EXTERNAL", "TRUE");
  String tablePath = FileUtils.hiveDirectoryName(url, topicsDir, tableName);
  table.setDataLocation(new Path(tablePath));
  table.setSerializationLib(getHiveParquetSerde());
  try {
    table.setInputFormatClass(getHiveParquetInputFormat());
    table.setOutputFormatClass(getHiveParquetOutputFormat());
  } catch (HiveException e) {
    throw new HiveMetaStoreException("Cannot find input/output format:", e);
  }
  // convert copycat schema schema to Hive columns
  List<FieldSchema> columns = HiveSchemaConverter.convertSchema(schema);
  table.setFields(columns);
  table.setPartCols(partitioner.partitionFields());
  return table;
}
 

@Test(expected=InvalidOperationException.class)
public void testRenamePartitionForUnknownTable() throws Exception {
  String dbName = testDB.getName();
  Table externalTable = getTestTable();
  externalTable.setTableType(TableType.EXTERNAL_TABLE.name());   
  
  StorageDescriptor sd = HiveToCatalogConverter.convertStorageDescriptor(testPartition.getSd());
  
  Partition oldPartition = new Partition()
      .withDatabaseName(dbName).withTableName(externalTable.getName())
      .withValues(Lists.newArrayList("oldval")).withStorageDescriptor(sd);
  
  Partition newPartition = new Partition()
                                    .withDatabaseName(dbName).withTableName(externalTable.getName())
                                    .withValues(Lists.newArrayList("newval")).withStorageDescriptor(sd);    
      
  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  doThrow(EntityNotFoundException.class).when(glueClient).getTable(any(GetTableRequest.class));
  
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(oldPartition));     
  
  metastoreClient.renamePartition(dbName, externalTable.getName(), oldPartition.getValues(),
      CatalogToHiveConverter.convertPartition(newPartition));
}
 

@Test(expected=InvalidOperationException.class)
public void testRenamePartitionForUnknownPartition() throws Exception {
  String dbName = testDB.getName();
  Table externalTable = getTestTable();
  externalTable.setTableType(TableType.EXTERNAL_TABLE.name());   
  
  StorageDescriptor sd = HiveToCatalogConverter.convertStorageDescriptor(testPartition.getSd());
  
  Partition oldPartition = new Partition()
      .withDatabaseName(dbName).withTableName(externalTable.getName())
      .withValues(Lists.newArrayList("oldval")).withStorageDescriptor(sd);
  
  Partition newPartition = new Partition()
                                    .withDatabaseName(dbName).withTableName(externalTable.getName())
                                    .withValues(Lists.newArrayList("newval")).withStorageDescriptor(sd);

  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(externalTable));
  doThrow(EntityNotFoundException.class).when(glueClient).getPartition(any(GetPartitionRequest.class));
  
  metastoreClient.renamePartition(dbName, externalTable.getName(), oldPartition.getValues(),
      CatalogToHiveConverter.convertPartition(newPartition));
}
 

@Test(expected=InvalidOperationException.class)
public void testRenamePartitionForInvalidSD() throws Exception {
  String dbName = testDB.getName();
  Table externalTable = getTestTable();
  externalTable.setTableType(TableType.EXTERNAL_TABLE.name());   
  
  StorageDescriptor sd = HiveToCatalogConverter.convertStorageDescriptor(testPartition.getSd());
  
  Partition oldPartition = new Partition()
      .withDatabaseName(dbName).withTableName(externalTable.getName())
      .withValues(Lists.newArrayList("oldval")).withStorageDescriptor(null);
  
  Partition newPartition = new Partition()
                                    .withDatabaseName(dbName).withTableName(externalTable.getName())
                                    .withValues(Lists.newArrayList("newval")).withStorageDescriptor(sd);

  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(externalTable));
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(oldPartition));

  metastoreClient.renamePartition(dbName, externalTable.getName(), oldPartition.getValues(),
      CatalogToHiveConverter.convertPartition(newPartition));
}
 

private Table constructAvroTable(String database, String tableName, Schema schema, Partitioner partitioner)
    throws HiveMetaStoreException {
  Table table = new Table(database, tableName);
  table.setTableType(TableType.EXTERNAL_TABLE);
  table.getParameters().put("EXTERNAL", "TRUE");
  String tablePath = FileUtils.hiveDirectoryName(url, topicsDir, tableName);
  table.setDataLocation(new Path(tablePath));
  table.setSerializationLib(avroSerde);
  try {
    table.setInputFormatClass(avroInputFormat);
    table.setOutputFormatClass(avroOutputFormat);
  } catch (HiveException e) {
    throw new HiveMetaStoreException("Cannot find input/output format:", e);
  }
  List<FieldSchema> columns = HiveSchemaConverter.convertSchema(schema);
  table.setFields(columns);
  table.setPartCols(partitioner.partitionFields());
  table.getParameters().put(AVRO_SCHEMA_LITERAL, avroData.fromConnectSchema(schema).toString());
  return table;
}
 

private Optional<SystemTable> getPropertiesSystemTable(ConnectorSession session, SchemaTableName tableName, SchemaTableName sourceTableName)
{
    Optional<Table> table = metastore.getTable(new HiveIdentity(session), sourceTableName.getSchemaName(), sourceTableName.getTableName());
    if (table.isEmpty() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
        throw new TableNotFoundException(tableName);
    }
    Map<String, String> sortedTableParameters = ImmutableSortedMap.copyOf(table.get().getParameters());
    List<ColumnMetadata> columns = sortedTableParameters.keySet().stream()
            .map(key -> new ColumnMetadata(key, VarcharType.VARCHAR))
            .collect(toImmutableList());
    List<Type> types = columns.stream()
            .map(ColumnMetadata::getType)
            .collect(toImmutableList());
    Iterable<List<Object>> propertyValues = ImmutableList.of(ImmutableList.copyOf(sortedTableParameters.values()));

    return Optional.of(createSystemTable(new ConnectorTableMetadata(sourceTableName, columns), constraint -> new InMemoryRecordSet(types, propertyValues).cursor()));
}
 

@Override
public Table getTable(final String dbName, final String tableName, boolean ignoreAuthzErrors) throws TException{

  Table table = getTableWithoutTableTypeChecking(dbName, tableName, ignoreAuthzErrors);

  if(table == null){
    return null;
  }

  TableType type = TableType.valueOf(table.getTableType());
  switch (type) {
    case EXTERNAL_TABLE:
    case MANAGED_TABLE:
      return table;

    case VIRTUAL_VIEW:
      throw UserException.unsupportedError().message("Hive views are not supported").build(NOPLogger.NOP_LOGGER);
    case INDEX_TABLE:
    default:
      return null;
  }
}
 

static Table createPartitionedTable(HiveMetaStoreClient metaStoreClient, String database, String table, File location)
  throws Exception {

  Table hiveTable = new Table();
  hiveTable.setDbName(database);
  hiveTable.setTableName(table);
  hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
  hiveTable.putToParameters("EXTERNAL", "TRUE");

  hiveTable.setPartitionKeys(PARTITION_COLUMNS);

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setLocation(location.toURI().toString());
  sd.setParameters(new HashMap<>());
  sd.setSerdeInfo(new SerDeInfo());

  hiveTable.setSd(sd);

  metaStoreClient.createTable(hiveTable);

  return hiveTable;
}
 

public TableMetadata(Table table, Map<String, HiveColumnStatistics> columnStatistics)
{
    owner = table.getOwner();
    tableType = table.getTableType();
    dataColumns = table.getDataColumns();
    partitionColumns = table.getPartitionColumns();
    parameters = table.getParameters();

    StorageFormat tableFormat = table.getStorage().getStorageFormat();
    storageFormat = Arrays.stream(HiveStorageFormat.values())
            .filter(format -> tableFormat.equals(StorageFormat.fromHiveStorageFormat(format)))
            .findFirst();
    bucketProperty = table.getStorage().getBucketProperty();
    serdeParameters = table.getStorage().getSerdeParameters();

    if (tableType.equals(TableType.EXTERNAL_TABLE.name())) {
        externalLocation = Optional.of(table.getStorage().getLocation());
    }
    else {
        externalLocation = Optional.empty();
    }

    viewOriginalText = table.getViewOriginalText();
    viewExpandedText = table.getViewExpandedText();
    this.columnStatistics = ImmutableMap.copyOf(requireNonNull(columnStatistics, "columnStatistics is null"));
}
 

static Table createUnpartitionedTable(
    HiveMetaStoreClient metaStoreClient,
    String database,
    String table,
    File location)
  throws TException {
  Table hiveTable = new Table();
  hiveTable.setDbName(database);
  hiveTable.setTableName(table);
  hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
  hiveTable.putToParameters("EXTERNAL", "TRUE");

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setLocation(location.toURI().toString());
  sd.setParameters(new HashMap<>());
  sd.setSerdeInfo(new SerDeInfo());

  hiveTable.setSd(sd);

  metaStoreClient.createTable(hiveTable);

  return hiveTable;
}
 

private static Table table(
        String location,
        List<Column> partitionColumns,
        Optional<HiveBucketProperty> bucketProperty,
        Map<String, String> tableParameters,
        StorageFormat storageFormat)
{
    Table.Builder tableBuilder = Table.builder();
    tableBuilder.getStorageBuilder()
            .setStorageFormat(storageFormat)
            .setLocation(location)
            .setSkewed(false)
            .setBucketProperty(bucketProperty);

    return tableBuilder
            .setDatabaseName("test_dbname")
            .setOwner("testOwner")
            .setTableName("test_table")
            .setTableType(TableType.MANAGED_TABLE.toString())
            .setDataColumns(ImmutableList.of(new Column("col1", HIVE_STRING, Optional.empty())))
            .setParameters(tableParameters)
            .setPartitionColumns(partitionColumns)
            .build();
}
 

/**
 * A base implementation for creating a non bucketed, external {@link HiveTable} for a {@link Path}.
 *
 * @param path a {@link Path} used to create the {@link HiveTable}.
 * @param dbName the database name for the created {@link HiveTable}.
 * @param tableName the table name for the created {@link HiveTable}.
 * @return a {@link HiveTable}s for the given {@link Path}.
 * @throws IOException
 */
protected HiveTable getTable(Path path, String dbName, String tableName) throws IOException {

  HiveTable.Builder tableBuilder = new HiveTable.Builder().withDbName(dbName).withTableName(tableName);

  if (!this.emptyInputPathFlag) {
    tableBuilder = tableBuilder.withSerdeManaager(HiveSerDeManager.get(this.props));
  }
  HiveTable table = tableBuilder.build();
  table.setLocation(this.fs.makeQualified(getTableLocation(path)).toString());

  if (!this.emptyInputPathFlag) {
    table.setSerDeProps(path);
  }

  // Setting table-level props.
  table.setProps(getRuntimePropsEnrichedTblProps());

  table.setStorageProps(this.props.getStorageProps());
  table.setSerDeProps(this.props.getSerdeProps());
  table.setNumBuckets(-1);
  table.setBucketColumns(Lists.<String> newArrayList());
  table.setTableType(TableType.EXTERNAL_TABLE.toString());
  return table;
}
 

private static Table createView(
    HiveMetaStoreClient metaStoreClient,
    String database,
    String view,
    String table,
    List<FieldSchema> partitionCols)
  throws TException {
  Table hiveView = new Table();
  hiveView.setDbName(database);
  hiveView.setTableName(view);
  hiveView.setTableType(TableType.VIRTUAL_VIEW.name());
  hiveView.setViewOriginalText(hql(database, table));
  hiveView.setViewExpandedText(expandHql(database, table, DATA_COLUMNS, partitionCols));
  hiveView.setPartitionKeys(partitionCols);

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setParameters(new HashMap<String, String>());
  sd.setSerdeInfo(new SerDeInfo());
  hiveView.setSd(sd);

  metaStoreClient.createTable(hiveView);

  return hiveView;
}
 

@Override
public Table getTable(String dbName, String tableName)
        throws TException
{
    accessCount.incrementAndGet();
    if (throwException) {
        throw new RuntimeException();
    }
    if (!dbName.equals(TEST_DATABASE) || !tableName.equals(TEST_TABLE)) {
        throw new NoSuchObjectException();
    }
    return new Table(
            TEST_TABLE,
            TEST_DATABASE,
            "",
            0,
            0,
            0,
            DEFAULT_STORAGE_DESCRIPTOR,
            ImmutableList.of(new FieldSchema("key", "string", null)),
            ImmutableMap.of(),
            "",
            "",
            TableType.MANAGED_TABLE.name());
}
 

@Override
public List<String> getViews(IMetaStoreClient client, String databaseName) throws UnknownDBException, TException {
	try {
		Method method = client.getClass().getMethod("getTables", String.class, String.class, TableType.class);
		return (List<String>) method.invoke(client, databaseName, null, TableType.VIRTUAL_VIEW);
	} catch (InvocationTargetException ite) {
		Throwable targetEx = ite.getTargetException();
		if (targetEx instanceof TException) {
			throw (TException) targetEx;
		} else {
			throw new CatalogException(String.format("Failed to get views for %s", databaseName), targetEx);
		}
	} catch (NoSuchMethodException | IllegalAccessException e) {
		throw new CatalogException(String.format("Failed to get views for %s", databaseName), e);
	}
}
 

private Table newTable() {
  Table table = new Table();
  table.setDbName(DB_NAME);
  table.setTableName(TABLE_NAME);
  table.setTableType(TableType.EXTERNAL_TABLE.name());

  StorageDescriptor sd = new StorageDescriptor();
  sd.setLocation(tableLocation);
  table.setSd(sd);

  HashMap<String, String> parameters = new HashMap<>();
  parameters.put(StatsSetupConst.ROW_COUNT, "1");
  table.setParameters(parameters);

  table.setPartitionKeys(PARTITIONS);
  return table;
}
 

@Test
public void testCreateTable() throws Exception {
	String location = warehouse + "/external_location";
	tableEnv.executeSql(String.format(
			"create external table tbl1 (d decimal(10,0),ts timestamp) partitioned by (p string) location '%s' tblproperties('k1'='v1')", location));
	Table hiveTable = hiveCatalog.getHiveTable(new ObjectPath("default", "tbl1"));
	assertEquals(TableType.EXTERNAL_TABLE.toString(), hiveTable.getTableType());
	assertEquals(1, hiveTable.getPartitionKeysSize());
	assertEquals(location, locationPath(hiveTable.getSd().getLocation()));
	assertEquals("v1", hiveTable.getParameters().get("k1"));
	assertFalse(hiveTable.getParameters().containsKey(SqlCreateHiveTable.TABLE_LOCATION_URI));

	tableEnv.executeSql("create table tbl2 (s struct<ts:timestamp,bin:binary>) stored as orc");
	hiveTable = hiveCatalog.getHiveTable(new ObjectPath("default", "tbl2"));
	assertEquals(TableType.MANAGED_TABLE.toString(), hiveTable.getTableType());
	assertEquals(OrcSerde.class.getName(), hiveTable.getSd().getSerdeInfo().getSerializationLib());
	assertEquals(OrcInputFormat.class.getName(), hiveTable.getSd().getInputFormat());
	assertEquals(OrcOutputFormat.class.getName(), hiveTable.getSd().getOutputFormat());

	tableEnv.executeSql("create table tbl3 (m map<timestamp,binary>) partitioned by (p1 bigint,p2 tinyint) " +
			"row format serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'");
	hiveTable = hiveCatalog.getHiveTable(new ObjectPath("default", "tbl3"));
	assertEquals(2, hiveTable.getPartitionKeysSize());
	assertEquals(LazyBinarySerDe.class.getName(), hiveTable.getSd().getSerdeInfo().getSerializationLib());

	tableEnv.executeSql("create table tbl4 (x int,y smallint) row format delimited fields terminated by '|' lines terminated by '\n'");
	hiveTable = hiveCatalog.getHiveTable(new ObjectPath("default", "tbl4"));
	assertEquals("|", hiveTable.getSd().getSerdeInfo().getParameters().get(serdeConstants.FIELD_DELIM));
	assertEquals("|", hiveTable.getSd().getSerdeInfo().getParameters().get(serdeConstants.SERIALIZATION_FORMAT));
	assertEquals("\n", hiveTable.getSd().getSerdeInfo().getParameters().get(serdeConstants.LINE_DELIM));

	tableEnv.executeSql("create table tbl5 (m map<bigint,string>) row format delimited collection items terminated by ';' " +
			"map keys terminated by ':'");
	hiveTable = hiveCatalog.getHiveTable(new ObjectPath("default", "tbl5"));
	assertEquals(";", hiveTable.getSd().getSerdeInfo().getParameters().get(serdeConstants.COLLECTION_DELIM));
	assertEquals(":", hiveTable.getSd().getSerdeInfo().getParameters().get(serdeConstants.MAPKEY_DELIM));
}
 

private void convertExistingReplicaTableToView() {
  when(mockReplicaLocationManager.getTableLocation()).thenReturn(null);
  when(mockReplicaLocationManager.getPartitionBaseLocation()).thenReturn(null);
  existingReplicaTable.setTableType(TableType.VIRTUAL_VIEW.name());
  existingReplicaTable.getSd().setLocation(null);
  existingPartition.getSd().setLocation(null);
}
 

@Before
public void before() throws TException, IOException {
  Table table = new Table();
  table.setDbName(DATABASE);
  table.setTableName("source_" + TABLE);
  table.setTableType(TableType.EXTERNAL_TABLE.name());
  table.putToParameters("EXTERNAL", "TRUE");

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(Arrays.asList(new FieldSchema("col1", "string", null)));
  sd.setSerdeInfo(new SerDeInfo());
  table.setSd(sd);

  hive.client().createTable(table);
}
 

@Test
public void viewReplicationModeIsMetadataMirror() {
  tableReplication.setReplicationMode(ReplicationMode.METADATA_MIRROR);
  when(table.getTableType()).thenReturn(TableType.VIRTUAL_VIEW.name());
  Replication replication = factory.newInstance(tableReplication);
  assertThat(replication, is(instanceOf(UnpartitionedTableMetadataMirrorReplication.class)));
}
 

@Test
public void newHiveTable() throws URISyntaxException {
  when(uriResolver.resolve(schema1, TABLE, 1))
      .thenReturn(new URI("https://s3.amazonaws.com/road-schema-bucket/roads/table/schemas/1/table_v1.avsc"));
  doReturn(Instant.ofEpochSecond(1526462225L)).when(clock).instant();

  Table result = underTest.newHiveTable(DATABASE, TABLE, PARTITION_COLUMN, LOCATION, schema1, 1);

  assertThat(result.getDbName(), is(DATABASE));
  assertThat(result.getTableName(), is(TABLE));
  assertThat(result.getTableType(), is(TableType.EXTERNAL_TABLE.toString()));
  Map<String, String> parameters = result.getParameters();
  assertThat(parameters.get("EXTERNAL"), is("TRUE"));
  assertThat(parameters.get("data-highway.version"), is(DataHighwayVersion.VERSION));
  assertThat(parameters.get("data-highway.last-revision"), is("2018-05-16T09:17:05Z"));
  assertThat(parameters.get(AvroHiveTableStrategy.AVRO_SCHEMA_URL),
      is("https://s3.amazonaws.com/road-schema-bucket/roads/table/schemas/1/table_v1.avsc"));
  assertThat(parameters.get(AvroHiveTableStrategy.AVRO_SCHEMA_VERSION), is("1"));
  List<FieldSchema> partitionKeys = result.getPartitionKeys();
  assertThat(partitionKeys.size(), is(1));
  assertThat(partitionKeys.get(0), is(new FieldSchema(PARTITION_COLUMN, "string", null)));
  StorageDescriptor storageDescriptor = result.getSd();
  assertThat(storageDescriptor.getInputFormat(), is(AvroStorageDescriptorFactory.AVRO_INPUT_FORMAT));
  assertThat(storageDescriptor.getOutputFormat(), is(AvroStorageDescriptorFactory.AVRO_OUTPUT_FORMAT));
  assertThat(storageDescriptor.getLocation(), is(LOCATION));
  assertThat(storageDescriptor.getCols().size(), is(0));
  SerDeInfo serdeInfo = storageDescriptor.getSerdeInfo();
  assertThat(serdeInfo.getSerializationLib(), is(AvroStorageDescriptorFactory.AVRO_SERDE));
}
 

@Test
public void newTable() {
  TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME,
      REPLICA_DATA_DESTINATION, FULL);
  Table replica = replicaAndStats.getTable();

  assertThat(replica.getDbName(), is(sourceTable.getDbName()));
  assertThat(replica.getTableName(), is(sourceTable.getTableName()));
  assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT));
  assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT));
  assertThat(replica.getSd().getLocation(), is(REPLICA_DATA_DESTINATION.toUri().toString()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(TABLE_LOCATION));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
  assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
  assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
  assertThat(replica.getTableType(), is(TableType.EXTERNAL_TABLE.name()));
  assertThat(replica.getParameters().get("EXTERNAL"), is("TRUE"));
  assertTrue(MetaStoreUtils.isExternalTable(replica));

  assertThat(replicaAndStats.getStatistics(), is(nullValue()));
}
 

@Test
public void newView() {
  sourceTableAndStats.getTable().setTableType(TableType.VIRTUAL_VIEW.name());
  sourceTableAndStats.getTable().getSd().setInputFormat(null);
  sourceTableAndStats.getTable().getSd().setOutputFormat(null);
  sourceTableAndStats.getTable().getSd().setLocation(null);

  TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME,
      null, FULL);
  Table replica = replicaAndStats.getTable();

  assertThat(replica.getDbName(), is(sourceTable.getDbName()));
  assertThat(replica.getTableName(), is(sourceTable.getTableName()));
  assertThat(replica.getSd().getInputFormat(), is(nullValue()));
  assertThat(replica.getSd().getOutputFormat(), is(nullValue()));
  assertThat(replica.getSd().getLocation(), is(nullValue()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(""));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
  assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
  assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
  assertThat(replica.getTableType(), is(TableType.VIRTUAL_VIEW.name()));
  assertTrue(MetaStoreUtils.isView(replica));

  assertThat(replicaAndStats.getStatistics(), is(nullValue()));
}
 

private void setReplicaTableType(Table source, Table replica) {
  if (TableType.VIRTUAL_VIEW.name().equals(source.getTableType())) {
    replica.setTableType(TableType.VIRTUAL_VIEW.name());
    return;
  }
  // We set the table to external no matter what. We don't want to delete data accidentally when dropping a mirrored
  // table.
  replica.setTableType(TableType.EXTERNAL_TABLE.name());
  replica.putToParameters(EXTERNAL, "TRUE");
}
 

@Test
public void newViewPartition() {
  sourceTable.setTableType(TableType.VIRTUAL_VIEW.name());
  sourceTable.getSd().setInputFormat(null);
  sourceTable.getSd().setOutputFormat(null);
  sourceTable.getSd().setLocation(null);

  sourcePartition.getSd().setInputFormat(null);
  sourcePartition.getSd().setOutputFormat(null);
  sourcePartition.getSd().setLocation(null);

  Partition replica = factory.newReplicaPartition(EVENT_ID, sourceTable, sourcePartition, DB_NAME, TABLE_NAME, null,
      FULL);

  assertThat(replica.getDbName(), is(sourceTable.getDbName()));
  assertThat(replica.getTableName(), is(sourceTable.getTableName()));
  assertThat(replica.getSd().getInputFormat(), is(nullValue()));
  assertThat(replica.getSd().getOutputFormat(), is(nullValue()));
  assertThat(replica.getSd().getLocation(), is(nullValue()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(""));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
}