Java Code Examples for org.apache.phoenix.util.SchemaUtil#getQualifiedTableName()

public void createDBObject() throws SQLException {
    try(Connection conn = DriverManager.getConnection(getUrl(), new Properties())) {
        schema = generateUniqueName();
        table = generateUniqueName();
        index = generateUniqueName();
        dataTableFullName = SchemaUtil.getQualifiedTableName(schema, table);
        indexTableFullName = SchemaUtil.getQualifiedTableName(schema, index);

        conn.createStatement().execute(String.format(CREATE_TABLE_DDL, dataTableFullName));
        conn.createStatement().execute(String.format(CREATE_INDEX_DDL, index, dataTableFullName));
        conn.commit();

        pconn = conn.unwrap(PhoenixConnection.class);
        pIndexTable = pconn.getTable(new PTableKey(pconn.getTenantId(), indexTableFullName));
        pDataTable = pconn.getTable(new PTableKey(pconn.getTenantId(), dataTableFullName));
    }
}
 

@Test
public void testGetSourceTableName_table() {
    String fullTableName = SchemaUtil.getQualifiedTableName(schema, tableName);
    PName sourcePhysicalName = SchemaUtil.getPhysicalHBaseTableName(schema, tableName,
            isNamespaceEnabled);
    String expectedName = SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName),
            isNamespaceEnabled).toString();
    //setup
    Mockito.when(inputTable.getType()).thenReturn(PTableType.TABLE);
    Mockito.when(inputTable.getPhysicalName()).thenReturn(sourcePhysicalName);
    Mockito.when(inputTable.getTableName()).thenReturn(PNameFactory.newName(tableName));
    Mockito.when(inputTable.getSchemaName()).thenReturn(PNameFactory.newName(schema));
    //test
    String output = IndexScrutinyMapper.getSourceTableName(inputTable, isNamespaceEnabled);
    //assert
    Assert.assertEquals(expectedName, output);
}
 

@Test
public void testGetSourceTableName_index() {
    String fullTableName = SchemaUtil.getQualifiedTableName(schema, indexName);
    PName sourcePhysicalName = SchemaUtil.getPhysicalHBaseTableName(schema, indexName,
            isNamespaceEnabled);
    String expectedName = SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName),
            isNamespaceEnabled).toString();

    //setup
    Mockito.when(inputTable.getType()).thenReturn(PTableType.INDEX);
    Mockito.when(inputTable.getPhysicalName()).thenReturn(sourcePhysicalName);
    Mockito.when(inputTable.getTableName()).thenReturn(PNameFactory.newName(indexName));
    Mockito.when(inputTable.getSchemaName()).thenReturn(PNameFactory.newName(schema));

    //test
    String output = IndexScrutinyMapper.getSourceTableName(inputTable, isNamespaceEnabled);
    //assert
    Assert.assertEquals(expectedName, output);
}
 

private void setupIndexAndDataTable(Connection connection) throws SQLException, IOException {
    pDataTable = PhoenixRuntime.getTableNoCache(connection, qDataTable);
    if (!isValidIndexTable(connection, qDataTable, indexTable, tenantId)) {
        throw new IllegalArgumentException(
                String.format(" %s is not an index table for %s for this connection",
                        indexTable, qDataTable));
    }
    pIndexTable = PhoenixRuntime.getTable(connection, schemaName != null && !schemaName.isEmpty()
            ? SchemaUtil.getQualifiedTableName(schemaName, indexTable) : indexTable);
    indexType = pIndexTable.getIndexType();
    if (schemaName != null && !schemaName.isEmpty()) {
        qIndexTable = SchemaUtil.getQualifiedTableName(schemaName, indexTable);
    } else {
        qIndexTable = indexTable;
    }
    if (IndexType.LOCAL.equals(indexType)) {
        isLocalIndexBuild = true;
        try (org.apache.hadoop.hbase.client.Connection hConn
                = getTemporaryHConnection(connection.unwrap(PhoenixConnection.class))) {
            RegionLocator regionLocator = hConn
                    .getRegionLocator(TableName.valueOf(pIndexTable.getPhysicalName().getBytes()));
            splitKeysBeforeJob = regionLocator.getStartKeys();
        }
    }
    // We have to mark Disable index to Building before we can set it to Active in the reducer. Otherwise it errors out with
    // index state transition error
    changeDisabledIndexStateToBuiding(connection);
}
 

@Test
public void testGetSourceTableName_view() {
    String fullTableName = SchemaUtil.getQualifiedTableName(schema, tableName);
    PName sourcePhysicalName = SchemaUtil.getPhysicalHBaseTableName(schema, tableName,
            isNamespaceEnabled);
    String expectedName = SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName),
            isNamespaceEnabled).toString();
    //setup
    Mockito.when(inputTable.getType()).thenReturn(PTableType.VIEW);
    Mockito.when(inputTable.getPhysicalName()).thenReturn(sourcePhysicalName);
    //test
    String output = IndexScrutinyMapper.getSourceTableName(inputTable, isNamespaceEnabled);
    //assert
    Assert.assertEquals(expectedName, output);
}
 

public Job createSubmittableJob(String schemaName, String indexTable, String dataTable,
        SourceTable sourceTable, Class<IndexScrutinyMapperForTest> mapperClass) throws Exception {
    Preconditions.checkArgument(SourceTable.DATA_TABLE_SOURCE.equals(sourceTable)
            || SourceTable.INDEX_TABLE_SOURCE.equals(sourceTable));

    final String qDataTable = SchemaUtil.getQualifiedTableName(schemaName, dataTable);
    final String qIndexTable;
    if (schemaName != null && !schemaName.isEmpty()) {
        qIndexTable = SchemaUtil.getQualifiedTableName(schemaName, indexTable);
    } else {
        qIndexTable = indexTable;
    }
    PhoenixConfigurationUtil.setScrutinyDataTable(configuration, qDataTable);
    PhoenixConfigurationUtil.setScrutinyIndexTable(configuration, qIndexTable);
    PhoenixConfigurationUtil.setScrutinySourceTable(configuration, sourceTable);
    PhoenixConfigurationUtil.setScrutinyOutputInvalidRows(configuration, outputInvalidRows);
    PhoenixConfigurationUtil.setScrutinyOutputMax(configuration, outputMaxRows);

    final PTable pdataTable = PhoenixRuntime.getTable(connection, qDataTable);
    final PTable pindexTable = PhoenixRuntime.getTable(connection, qIndexTable);

    // set CURRENT_SCN for our scan so that incoming writes don't throw off scrutiny
    configuration.set(PhoenixConfigurationUtil.CURRENT_SCN_VALUE, Long.toString(ts));

    // set the source table to either data or index table
    SourceTargetColumnNames columnNames =
            SourceTable.DATA_TABLE_SOURCE.equals(sourceTable)
                    ? new SourceTargetColumnNames.DataSourceColNames(pdataTable,
                            pindexTable)
                    : new SourceTargetColumnNames.IndexSourceColNames(pdataTable,
                            pindexTable);
    String qSourceTable = columnNames.getQualifiedSourceTableName();
    List<String> sourceColumnNames = columnNames.getSourceColNames();
    List<String> sourceDynamicCols = columnNames.getSourceDynamicCols();
    List<String> targetDynamicCols = columnNames.getTargetDynamicCols();

    // Setup the select query against source - we either select the index columns from the
    // index table,
    // or select the data table equivalents of the index columns from the data table
    final String selectQuery =
            QueryUtil.constructSelectStatement(qSourceTable, sourceColumnNames, null,
                Hint.NO_INDEX, true);
    LOGGER.info("Query used on source table to feed the mapper: " + selectQuery);

    PhoenixConfigurationUtil.setScrutinyOutputFormat(configuration, outputFormat);
    // if outputting to table, setup the upsert to the output table
    if (outputInvalidRows && OutputFormat.TABLE.equals(outputFormat)) {
        String upsertStmt =
                IndexScrutinyTableOutput.constructOutputTableUpsert(sourceDynamicCols,
                    targetDynamicCols, connection);
        PhoenixConfigurationUtil.setUpsertStatement(configuration, upsertStmt);
        LOGGER.info("Upsert statement used for output table: " + upsertStmt);
    }

    final String jobName =
            String.format(INDEX_JOB_NAME_TEMPLATE, qSourceTable,
                columnNames.getQualifiedTargetTableName());
    final Job job = Job.getInstance(configuration, jobName);

    if (!useSnapshot) {
        PhoenixMapReduceUtil.setInput(job, PhoenixIndexDBWritable.class, qDataTable,
            selectQuery);
    } else { // TODO check if using a snapshot works
        Admin admin = null;
        String snapshotName;
        try {
            final PhoenixConnection pConnection =
                    connection.unwrap(PhoenixConnection.class);
            admin = pConnection.getQueryServices().getAdmin();
            String pdataTableName = pdataTable.getName().getString();
            snapshotName = new StringBuilder(pdataTableName).append("-Snapshot").toString();
            admin.snapshot(snapshotName, TableName.valueOf(pdataTableName));
        } finally {
            if (admin != null) {
                admin.close();
            }
        }
        // root dir not a subdirectory of hbase dir
        Path rootDir = new Path("hdfs:///index-snapshot-dir");
        FSUtils.setRootDir(configuration, rootDir);

        // set input for map reduce job using hbase snapshots
        //PhoenixMapReduceUtil.setInput(job, PhoenixIndexDBWritable.class, snapshotName,
        //    qDataTable, restoreDir, selectQuery);
    }
    TableMapReduceUtil.initCredentials(job);
    Path outputPath =
            getOutputPath(configuration, basePath,
                SourceTable.DATA_TABLE_SOURCE.equals(sourceTable) ? pdataTable
                        : pindexTable);

    return configureSubmittableJob(job, outputPath, mapperClass);
}
 

@VisibleForTesting
public int populateIndexToolAttributes(CommandLine cmdLine) throws Exception {
    boolean useTenantId = cmdLine.hasOption(TENANT_ID_OPTION.getOpt());
    boolean useStartTime = cmdLine.hasOption(START_TIME_OPTION.getOpt());
    boolean useEndTime = cmdLine.hasOption(END_TIME_OPTION.getOpt());
    boolean retryVerify = cmdLine.hasOption(RETRY_VERIFY_OPTION.getOpt());
    boolean verify = cmdLine.hasOption(VERIFY_OPTION.getOpt());
    boolean disableLogging = cmdLine.hasOption(DISABLE_LOGGING_OPTION.getOpt());

    if (useTenantId) {
        tenantId = cmdLine.getOptionValue(TENANT_ID_OPTION.getOpt());
    }
    if(useStartTime) {
        startTime = new Long(cmdLine.getOptionValue(START_TIME_OPTION.getOpt()));
    }
    if (useEndTime) {
        endTime = new Long(cmdLine.getOptionValue(END_TIME_OPTION.getOpt()));
    }
    if(retryVerify) {
        lastVerifyTime = new Long(cmdLine.getOptionValue(RETRY_VERIFY_OPTION.getOpt()));
        validateLastVerifyTime();
    }
    if(isTimeRangeSet(startTime, endTime)) {
        validateTimeRange();
    }
    if (verify) {
        String value = cmdLine.getOptionValue(VERIFY_OPTION.getOpt());
        indexVerifyType = IndexVerifyType.fromValue(value);
        if (disableLogging) {
            disableLoggingType =
                IndexDisableLoggingType.fromValue(
                    cmdLine.getOptionValue(DISABLE_LOGGING_OPTION.getOpt()));
        }
    }
    schemaName = cmdLine.getOptionValue(SCHEMA_NAME_OPTION.getOpt());
    dataTable = cmdLine.getOptionValue(DATA_TABLE_OPTION.getOpt());
    indexTable = cmdLine.getOptionValue(INDEX_TABLE_OPTION.getOpt());
    isPartialBuild = cmdLine.hasOption(PARTIAL_REBUILD_OPTION.getOpt());
    qDataTable = SchemaUtil.getQualifiedTableName(schemaName, dataTable);
    basePath = cmdLine.getOptionValue(OUTPUT_PATH_OPTION.getOpt());
    isForeground = cmdLine.hasOption(RUN_FOREGROUND_OPTION.getOpt());
    useSnapshot = cmdLine.hasOption(SNAPSHOT_OPTION.getOpt());
    shouldDeleteBeforeRebuild = cmdLine.hasOption(DELETE_ALL_AND_REBUILD_OPTION.getOpt());
    return 0;
}
 

public String getQualifiedDataTableName() {
    return SchemaUtil.getQualifiedTableName(pdataTable.getSchemaName().getString(),
        pdataTable.getTableName().getString());
}
 

public String getQualifiedIndexTableName() {
    return SchemaUtil.getQualifiedTableName(pindexTable.getSchemaName().getString(),
        pindexTable.getTableName().getString());
}
 

@Test
public void testSecondaryGlobalIndexFailure() throws Exception {
    String schemaName = generateUniqueName();
    String dataTableName = generateUniqueName();
    String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
    String indexTableName = generateUniqueName();
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
        String stmString1 =
                "CREATE TABLE " + dataTableFullName
                        + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
                        + tableDDLOptions;
        conn.createStatement().execute(stmString1);
        String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
        PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);

        // Insert two rows
        IndexToolIT.upsertRow(stmt1, 1);
        IndexToolIT.upsertRow(stmt1, 2);
        conn.commit();

        String stmtString2 =
                String.format(
                        "CREATE INDEX %s ON %s  (LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz') ASYNC ", indexTableName, dataTableFullName);
        conn.createStatement().execute(stmtString2);

        // Run the index MR job.
        IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName);

        String qIndexTableName = SchemaUtil.getQualifiedTableName(schemaName, indexTableName);

        // Verify that the index table is in the ACTIVE state
        assertEquals(PIndexState.ACTIVE, TestUtil.getIndexState(conn, qIndexTableName));

        ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices();
        Admin admin = queryServices.getAdmin();
        TableName tableName = TableName.valueOf(qIndexTableName);
        admin.disableTable(tableName);

        // Run the index MR job and it should fail (return -1)
        IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
                null, -1, new String[0]);

        // Verify that the index table should be still in the ACTIVE state
        assertEquals(PIndexState.ACTIVE, TestUtil.getIndexState(conn, qIndexTableName));
    }
}