Java Code Examples for org.apache.phoenix.util.SchemaUtil#getTableName()

/**
 * Ensure that we cannot add a column to a base table if QueryServices.BLOCK_METADATA_CHANGES_REQUIRE_PROPAGATION
 * is true
 */
@Test
public void testAddingColumnFails() throws Exception {
    try (Connection conn = DriverManager.getConnection(getJdbcUrl())) {
        String fullTableName = SchemaUtil.getTableName(generateUniqueName(), generateUniqueName());
        String fullViewName = SchemaUtil.getTableName(generateUniqueName(), generateUniqueName());
        String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, v1 INTEGER " +
                "CONSTRAINT pk PRIMARY KEY (k1))";
        conn.createStatement().execute(ddl);

        ddl = "CREATE VIEW " + fullViewName + " AS SELECT * FROM " + fullTableName;
        conn.createStatement().execute(ddl);

        try {
            ddl = "ALTER TABLE " + fullTableName + " ADD v2 INTEGER";
            conn.createStatement().execute(ddl);
            fail();
        }
        catch (SQLException e) {
            assertEquals(SQLExceptionCode.CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode());
        }
    }
}
 

private void createBaseTableIndexAndViews(Connection baseTableConnection, String baseTableFullName,
                                          Connection viewConnection, String childViewSchemaName,
                                          String grandchildViewSchemaName, String grandGrandChildViewSchemaName)
        throws SQLException {
    baseTableConnection.createStatement().execute(generateDDL(String.format(createBaseTableFirstPartDDL,
            baseTableFullName) + createBaseTableSecondPartDDL));
    baseTableConnection.createStatement().execute(String.format(createBaseTableIndexDDL,
            generateUniqueName(), baseTableFullName));
    // Create a view tree (i.e., tree of views) with depth of 3
    for (int i = 0; i < fanout; i++) {
        String childView = SchemaUtil.getTableName(childViewSchemaName, generateUniqueName());
        viewConnection.createStatement().execute(String.format(createViewDDL, childView, baseTableFullName));
        for (int j = 0; j < fanout; j++) {
            String grandchildView = SchemaUtil.getTableName(grandchildViewSchemaName, generateUniqueName());
            viewConnection.createStatement().execute(String.format(createViewDDL, grandchildView, childView));
            for (int k = 0; k < fanout; k++) {
                viewConnection.createStatement().execute(String.format(createViewDDL,
                        SchemaUtil.getTableName(grandGrandChildViewSchemaName, generateUniqueName()),
                        grandchildView));
            }
        }
    }
}
 

@Test
public void testDeletePhysicalTableLinks() throws Exception {
    String baseTableName = generateUniqueName();
    String baseTableFullName = SchemaUtil.getTableName(SCHEMA1, baseTableName);
    try (Connection connection = DriverManager.getConnection(getUrl());
         Connection viewConnection =
                 isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL) : connection) {
        createBaseTableIndexAndViews(connection, baseTableFullName, viewConnection, SCHEMA2, SCHEMA3, SCHEMA3);
        // Delete the physical table link rows from the system catalog
        executeDeleteQuery(connection, deletePhysicalLinks, SCHEMA2);
        // Verify that the views we have created are still in the system catalog table
        verifyCountQuery(connection, countViewsQuery, SCHEMA2, childCount);
        verifyCountQuery(connection, countViewsQuery, SCHEMA3, grandChildCount + grandGrandChildCount);
        // Run the orphan view tool to remove orphan views
        runOrphanViewTool(true, false, true, false);
        // Verify that the orphan views have been removed
        verifyLineCount(viewFileName, childCount + grandChildCount + grandGrandChildCount);
        verifyNoViewNoLinkInSystemCatalog(connection, SCHEMA2);
        verifyNoViewNoLinkInSystemCatalog(connection, SCHEMA3);
        // Verify that there there is no link in the system child link table
        verifyNoChildLink(connection, SCHEMA1);
        verifyNoChildLink(connection, SCHEMA2);
        verifyNoChildLink(connection, SCHEMA3);
        deleteAllRows(connection, SCHEMA1, SCHEMA2, SCHEMA3, SCHEMA4);
    }
}
 

@Test
public void testDeleteBaseTableRows() throws Exception {
    String baseTableName = generateUniqueName();
    String baseTableFullName = SchemaUtil.getTableName(SCHEMA1, baseTableName);
    try (Connection connection = DriverManager.getConnection(getUrl());
         Connection viewConnection =
                 isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL) : connection) {
        createBaseTableIndexAndViews(connection, baseTableFullName, viewConnection, SCHEMA2, SCHEMA2, SCHEMA2);
        // Delete the base table row from the system catalog
        executeDeleteQuery(connection, deleteTableRows, SCHEMA1);
        // Verify that the views we have created are still in the system catalog table
        ResultSet rs = connection.createStatement().executeQuery(countAllViewsQuery);
        assertTrue(rs.next());
        assertTrue(rs.getLong(1) == childCount + grandChildCount + grandGrandChildCount);
        // Run the orphan view tool to identify orphan views
        runOrphanViewTool(false, true, true, false);
        verifyOrphanFileLineCounts(childCount + grandChildCount + grandGrandChildCount,
                0,
                childCount + grandChildCount + grandGrandChildCount,
                childCount);
        // Verify that orphan views have not yet dropped as we just identified them
        rs = connection.createStatement().executeQuery(countAllViewsQuery);
        assertTrue(rs.next());
        assertTrue(rs.getLong(1) == childCount + grandChildCount + grandGrandChildCount);
        // Drop the previously identified orphan views
        runOrphanViewTool(true, false, false, true);
        // Verify that the orphan views and links have been removed from the system catalog table
        verifyNoViewNoLinkInSystemCatalog(connection, SCHEMA2);
        // Verify that there there is no link in the system child link table
        verifyNoChildLink(connection, SCHEMA1);
        deleteAllRows(connection, SCHEMA1, SCHEMA2, SCHEMA3, SCHEMA4);
    }
}
 

@Test
@Repeat(5)
public void testDeleteAndUpsertAfterFailure() throws Throwable {
    final int nRows = 10;
    String schemaName = generateUniqueName();
    String tableName = generateUniqueName();
    String indexName = generateUniqueName();
    final String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
    String fullIndexName = SchemaUtil.getTableName(schemaName, indexName);
    try (Connection conn = DriverManager.getConnection(getUrl())) {
        conn.createStatement().execute("CREATE TABLE " + fullTableName + "(k INTEGER PRIMARY KEY, v1 INTEGER, v2 INTEGER) COLUMN_ENCODED_BYTES = 0, STORE_NULLS=true");
        conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
        mutateRandomly(conn, fullTableName, nRows);
        long disableTS = EnvironmentEdgeManager.currentTimeMillis();
        Table metaTable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES);
        IndexUtil.updateIndexState(fullIndexName, disableTS, metaTable, PIndexState.DISABLE);
        boolean[] cancel = new boolean[1];
        try {
            runIndexRebuilderAsync(500,cancel,fullTableName);
            mutateRandomly(conn, fullTableName, nRows);
            TestUtil.waitForIndexRebuild(conn, fullIndexName, PIndexState.ACTIVE);
        } finally {
            cancel[0] = true;
        }
        
        long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, fullTableName, fullIndexName);
        assertEquals(nRows,actualRowCount);
   }
}
 

@Before
public void beforeTest() throws SQLException {
    String schemaName = generateUniqueName();
    String tableName = "T_" + generateUniqueName();
    fullTableName = SchemaUtil.getTableName(schemaName, tableName);
    physicalTableName = SchemaUtil.getPhysicalHBaseTableName(schemaName,
            tableName, false).getString();
    conn = getConnection();
    conn.createStatement().execute(
            "CREATE TABLE " + fullTableName + " ( k VARCHAR, a_string_array VARCHAR(100) ARRAY[4],"
                    + " b_string_array VARCHAR(100) ARRAY[4] \n"
                    + " CONSTRAINT pk PRIMARY KEY (k, b_string_array DESC)) GUIDE_POSTS_WIDTH = 10");
    upsertValues(conn, fullTableName);
}
 

@Ignore
@Test
public void testCreateIndexDuringUpsertSelect() throws Exception {
    // This test times out at the UPSERT SELECT call for local index
    if (localIndex) { // TODO: remove after PHOENIX-3314 is fixed 
        return;
    }
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    String tableName = "TBL_" + generateUniqueName();
    String indexName = "IND_" + generateUniqueName();
    String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
    TABLE_NAME = fullTableName;
    String ddl ="CREATE TABLE " + TABLE_NAME + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions;
    INDEX_DDL = "CREATE " + (localIndex ? "LOCAL" : "") + " INDEX IF NOT EXISTS " + indexName + " ON " + TABLE_NAME
            + " (long_pk, varchar_pk)"
            + " INCLUDE (long_col1, long_col2)";

    try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
        conn.setAutoCommit(false);
        Statement stmt = conn.createStatement();
        stmt.execute(ddl);

        upsertRows(conn, TABLE_NAME, 220);
        conn.commit();

        // run the upsert select and also create an index
        conn.setAutoCommit(true);
        String upsertSelect = "UPSERT INTO " + TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) " +
                "SELECT varchar_pk||'_upsert_select', char_pk, int_pk, long_pk, decimal_pk, date_pk FROM "+ TABLE_NAME;
        conn.createStatement().execute(upsertSelect);
        TestUtil.waitForIndexRebuild(conn, indexName, PIndexState.ACTIVE);
        ResultSet rs;
        rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ COUNT(*) FROM " + TABLE_NAME);
        assertTrue(rs.next());
        assertEquals(440,rs.getInt(1));
        rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + indexName);
        assertTrue(rs.next());
        assertEquals(440,rs.getInt(1));
    }
}
 

private void doTestQueryBackToDataTableWithDescPKColumn(boolean isSecondPKDesc) throws SQLException {
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    String tableName = "TBL_" + generateUniqueName();
    String indexName = "IND_" + generateUniqueName();
    String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
    String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);

    try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
        // create data table and index table
        conn.setAutoCommit(true);
        Statement stmt = conn.createStatement();
        String ddl = "CREATE TABLE " + fullTableName + "(p1 integer not null, p2 integer not null, " +
                " a integer, b integer CONSTRAINT PK PRIMARY KEY ";
        if (isSecondPKDesc) {
            ddl += "(p1, p2 desc))";
        } else {
            ddl += "(p1 desc, p2))";
        }
        stmt.executeUpdate(ddl);
        ddl = "CREATE "+ (localIndex ? "LOCAL " : "") + " INDEX " + indexName + " on " + fullTableName + "(a)";
        stmt.executeUpdate(ddl);

        // upsert a single row
        String upsert = "UPSERT INTO " + fullTableName + " VALUES(1,2,3,4)";
        stmt.executeUpdate(upsert);

        // try select with index
        // a = 3, should hit index table, but we select column B, so it will query back to data table
        String query = "SELECT /*+index(" + fullTableName + " " + fullIndexName + "*/ b from " + fullTableName +
                " WHERE a = 3";
        ResultSet rs = stmt.executeQuery(query);
        assertTrue(rs.next());
        assertEquals(4, rs.getInt(1));
        assertFalse(rs.next());
        rs.close();
        stmt.close();
    }
}
 

private static void assertNoActiveIndex(Connection conn, String schemaName, String tableName) throws SQLException {
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
    conn.createStatement().executeQuery("SELECT count(*) FROM " + fullTableName).next(); // client side cache will update
    PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
    pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName)).getIndexMaintainers(ptr, pconn);
    assertTrue(ptr.getLength() == 0);
}
 

@Override
public TableRef resolveTable(String schemaName, String tableName) throws SQLException {
    String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
    List<TableRef> tableRefs = tableMap.get(fullTableName);
    if (tableRefs.size() == 0) {
        throw new TableNotFoundException(fullTableName);
    } else if (tableRefs.size() > 1) {
        throw new AmbiguousTableException(tableName);
    } else {
        return tableRefs.get(0);
    }
}
 

@Test
public void testReadOnlyViewWithCaseSensitiveColumnNames() throws Exception {
    Connection conn = DriverManager.getConnection(getUrl());
    String fullTableName = SchemaUtil.getTableName(SCHEMA1, generateUniqueName());
    String viewName = SchemaUtil.getTableName(SCHEMA2, generateUniqueName());
    String ddl = "CREATE TABLE " + fullTableName + " (\"k\" INTEGER NOT NULL PRIMARY KEY, \"v1\" INTEGER, \"a\".v2 VARCHAR)" + tableDDLOptions;

    conn.createStatement().execute(ddl);
    ddl = "CREATE VIEW " + viewName + " (v VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE \"k\" > 5 and \"v1\" > 1";
    conn.createStatement().execute(ddl);
    
    try {
        conn.createStatement().execute("UPSERT INTO " + viewName + " VALUES(1)");
        fail();
    } catch (ReadOnlyTableException e) {
        
    }
    for (int i = 0; i < 10; i++) {
        conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(" + i + ", " + (i+10) + ",'A')");
    }
    conn.commit();
    
    int count = 0;
    ResultSet rs = conn.createStatement().executeQuery("SELECT \"k\", \"v1\",\"a\".v2 FROM " + viewName);
    while (rs.next()) {
        count++;
        assertEquals(count + 5, rs.getInt(1));
    }
    assertEquals(4, count);
}
 

@Test
public void testDeleteFromNonPKColumnIndex() throws Exception {
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    String tableName = "TBL_" + generateUniqueName();
    String indexName = "IND_" + generateUniqueName();
    String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
    String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);

    String ddl ="CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions;
    try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
        conn.setAutoCommit(false);
        Statement stmt = conn.createStatement();
        stmt.execute(ddl);
        BaseTest.populateTestTable(fullTableName);
        ddl = "CREATE " + (localIndex ? "LOCAL" : "") + " INDEX " + indexName + " ON " + fullTableName
                    + " (long_col1, long_col2)"
                    + " INCLUDE (decimal_col1, decimal_col2)";
        stmt.execute(ddl);
    }
    try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
        ResultSet rs;

        rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullTableName);
        assertTrue(rs.next());
        assertEquals(3,rs.getInt(1));
        rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullIndexName);
        assertTrue(rs.next());
        assertEquals(3,rs.getInt(1));

        String dml = "DELETE from " + fullTableName + " WHERE long_col2 = 4";
        assertEquals(1,conn.createStatement().executeUpdate(dml));
        assertNoClientSideIndexMutations(conn);
        conn.commit();

        // query the data table
        String query = "SELECT /*+ NO_INDEX */ long_pk FROM " + fullTableName;
        rs = conn.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertEquals(1L, rs.getLong(1));
        assertTrue(rs.next());
        assertEquals(3L, rs.getLong(1));
        assertFalse(rs.next());

        // query the index table
        query = "SELECT long_pk FROM " + fullTableName + " ORDER BY long_col1";
        rs = conn.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertEquals(1L, rs.getLong(1));
        assertTrue(rs.next());
        assertEquals(3L, rs.getLong(1));
        assertFalse(rs.next());

        conn.createStatement().execute("DROP INDEX " + indexName + " ON " + fullTableName);
    }
}
 

@Test
public void testGlobalImmutableIndexCreate() throws Exception {
    if (localIndex || transactionProvider != null) {
        return;
    }
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);

    ArrayList<String> immutableStorageProps = new ArrayList<String>();
    immutableStorageProps.add(null);
    if (!tableDDLOptions.contains(IMMUTABLE_STORAGE_SCHEME)) {
       immutableStorageProps.add(SINGLE_CELL_ARRAY_WITH_OFFSETS.toString());
    }
    try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
        conn.setAutoCommit(true);
        for (String storageProp : immutableStorageProps) {
            String tableName = "TBL_" + generateUniqueName();
            String indexName = "IND_" + generateUniqueName();
            String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
            String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
            TABLE_NAME = fullTableName;
            int numRows = 1;
            createAndPopulateTableAndIndexForConsistentIndex(conn, fullTableName, fullIndexName,
                    numRows, storageProp);

            ResultSet rs;
            rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ COUNT(*) FROM " + TABLE_NAME);
            assertTrue(rs.next());
            assertEquals(numRows, rs.getInt(1));
            rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullIndexName);
            assertTrue(rs.next());
            assertEquals(numRows, rs.getInt(1));
            assertEquals(true, verifyRowsForEmptyColValue(conn, fullIndexName,
                    IndexRegionObserver.VERIFIED_BYTES));
            rs = conn.createStatement().executeQuery("SELECT * FROM " + fullIndexName);
            assertTrue(rs.next());
            assertEquals("1", rs.getString(1));

            // Now try to fail Phase1 and observe that index state is not DISABLED
            try (Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();) {
                admin.disableTable(TableName.valueOf(fullIndexName));
                boolean isWriteOnDisabledIndexFailed = false;
                try {
                    upsertRows(conn, fullTableName, numRows);
                } catch (SQLException ex) {
                    isWriteOnDisabledIndexFailed = true;
                }
                assertEquals(true, isWriteOnDisabledIndexFailed);
                PIndexState indexState = TestUtil.getIndexState(conn, fullIndexName);
                assertEquals(PIndexState.ACTIVE, indexState);

            }
        }
    }
}
 

@Test
public void testOverrideIndexRebuildPageSizeFromIndexTool() throws Exception {
    String schemaName = generateUniqueName();
    String dataTableName = generateUniqueName();
    String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
    String indexTableName = generateUniqueName();
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);

    try(Connection conn = DriverManager.getConnection(getUrl(), props)) {
        String stmString1 =
            "CREATE TABLE " + dataTableFullName
                + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
                + tableDDLOptions;
        conn.createStatement().execute(stmString1);
        String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
        PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);

        // Insert NROWS rows
        final int NROWS = 16;
        for (int i = 0; i < NROWS; i++) {
            IndexToolIT.upsertRow(stmt1, i);
        }
        conn.commit();

        String stmtString2 =
                String.format(
                        "CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP) ASYNC ", indexTableName, dataTableFullName);
        conn.createStatement().execute(stmtString2);

        // Run the index MR job and verify that the index table is built correctly
        Configuration conf = new Configuration(getUtility().getConfiguration());
        conf.set(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(2));
        IndexTool indexTool = IndexToolIT.runIndexTool(conf, directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BEFORE, new String[0]);
        assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
        assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
        assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
        assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
        assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
        assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
        assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
    }
}
 

private static String generateTableNameWithSchema() {
    return SchemaUtil.getTableName(SCHEMA_NAME, generateUniqueName());
}
 

@Test
public void testIndexStateOnException() throws Exception {
    if (localIndex  || useSnapshot || useViewIndex) {
        return;
    }

    String schemaName = generateUniqueName();
    String dataTableName = generateUniqueName();
    String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
    String indexTableName = generateUniqueName();
    String indexFullName = SchemaUtil.getTableName(schemaName, indexTableName);
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    try (Connection conn = DriverManager.getConnection(getUrl(), props)){
        Statement stmt = conn.createStatement();
        stmt.execute(String.format(
                "CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) %s",
                dataTableFullName, tableDDLOptions));

        stmt.execute(String.format(
                "UPSERT INTO %s VALUES(1, 'Phoenix', 12345)", dataTableFullName));

        conn.commit();

        // Configure IndexRegionObserver to fail the first write phase. This should not
        // lead to any change on index and thus index verify during index rebuild should fail
        IndexRegionObserver.setIgnoreIndexRebuildForTesting(true);
        stmt.execute(String.format(
                "CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP) ASYNC",
                indexTableName, dataTableFullName));

        // Verify that the index table is not in the ACTIVE state
        assertFalse(checkIndexState(conn, indexFullName, PIndexState.ACTIVE, 0L));

        // Run the index MR job and verify that the index table rebuild fails
        IndexToolIT.runIndexTool(true, false, schemaName, dataTableName,
                indexTableName, null, -1, IndexTool.IndexVerifyType.AFTER);

        IndexRegionObserver.setIgnoreIndexRebuildForTesting(false);

        // job failed, verify that the index table is still not in the ACTIVE state
        assertFalse(checkIndexState(conn, indexFullName, PIndexState.ACTIVE, 0L));

        // Run the index MR job and verify that the index table rebuild succeeds
        IndexToolIT.runIndexTool(true, false, schemaName, dataTableName,
                indexTableName, null, 0, IndexTool.IndexVerifyType.AFTER);

        // job passed, verify that the index table is in the ACTIVE state
        assertTrue(checkIndexState(conn, indexFullName, PIndexState.ACTIVE, 0L));
    }
}
 

/**
 * Tests that the index rebuilder retries for exactly the configured # of retries
 * @throws Exception
 */
@Test
public void testRebuildRetriesSuccessful() throws Throwable {
    int numberOfRetries = 5;
    Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(10);
    serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_ATTRIB, Boolean.TRUE.toString());
    serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, Long.toString(REBUILD_INTERVAL));
    serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, "50000000");
    serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_PERIOD, Long.toString(REBUILD_PERIOD)); // batch at 50 seconds
    serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, Long.toString(WAIT_AFTER_DISABLED));
    serverProps.put(HConstants.HBASE_CLIENT_RETRIES_NUMBER, numberOfRetries + "");
    Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(1);
    // Index rebuilds are not needed with IndexRegionObserver
    clientProps.put(QueryServices.INDEX_REGION_OBSERVER_ENABLED_ATTRIB, "false");
    setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
    indexRebuildTaskRegionEnvironment =
            getUtility()
                     .getRSForFirstRegionInTable(
                         PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME)
                     .getRegions(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME)
                     .get(0).getCoprocessorHost()
                     .findCoprocessorEnvironment(MetaDataRegionObserver.class.getName());
    MetaDataRegionObserver.initRebuildIndexConnectionProps(
        indexRebuildTaskRegionEnvironment.getConfiguration());
    try (Connection conn = DriverManager.getConnection(getUrl())) {
        String schemaName = generateUniqueName();
        String tableName = generateUniqueName();
        String indexName = generateUniqueName();
        final String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
        final String fullIndexName = SchemaUtil.getTableName(schemaName, indexName);
        conn.createStatement().execute("CREATE TABLE " + fullTableName + "(k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR, v3 VARCHAR) DISABLE_INDEX_ON_WRITE_FAILURE = TRUE");
        conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + fullTableName + " (v1, v2)");
        Table metaTable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES);
        IndexUtil.updateIndexState(fullIndexName, EnvironmentEdgeManager.currentTimeMillis(), metaTable, PIndexState.DISABLE);
        conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES('a','a','0')");
        conn.commit();
        // Simulate write failure when rebuilder runs
        TestUtil.addCoprocessor(conn, fullIndexName, WriteFailingRegionObserver.class);
        waitForIndexState(conn, fullTableName, fullIndexName, PIndexState.INACTIVE);
        long pendingDisableCount = TestUtil.getPendingDisableCount(
                conn.unwrap(PhoenixConnection.class), fullIndexName);
        // rebuild writes should retry for exactly the configured number of times
        ExecutorService executor = Executors.newSingleThreadExecutor();
        try {
            Future<Boolean> future = executor.submit(new Callable<Boolean>() {
                @Override
                public Boolean call() throws Exception {
                    runIndexRebuilder(fullTableName);
                    return true;
                }});
            assertTrue(future.get(120, TimeUnit.SECONDS));
            assertEquals(numberOfRetries, WriteFailingRegionObserver.attempts.get());
            // Index rebuild write failures should not increase the pending disable count of the index table
            assertEquals(pendingDisableCount, TestUtil.getPendingDisableCount(
                    conn.unwrap(PhoenixConnection.class), fullIndexName));
        } finally {
            executor.shutdownNow();
        }
    }
}
 

@Ignore
@Test
public void testCreateIndexWhileUpsertingData() throws Exception {
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    String tableName = "TBL_" + generateUniqueName();
    String indexName = "IND_" + generateUniqueName();
    String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
    String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
    String ddl ="CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions;
    String indexDDL = "CREATE " + (localIndex ? "LOCAL" : "") + " INDEX " + indexName + " ON " + fullTableName
            + " (long_pk, varchar_pk)"
            + " INCLUDE (long_col1, long_col2)";
    int numThreads = 2;
    ExecutorService executorService = Executors.newFixedThreadPool(numThreads, new ThreadFactory() {
        @Override
        public Thread newThread(Runnable r) {
            Thread t = Executors.defaultThreadFactory().newThread(r);
            t.setDaemon(true);
            t.setPriority(Thread.MIN_PRIORITY);
            return t;
        }
    });
    try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
        conn.setAutoCommit(true);
        Statement stmt = conn.createStatement();
        stmt.execute(ddl);

        ResultSet rs;
        rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ COUNT(*) FROM " + fullTableName);
        assertTrue(rs.next());
        int dataTableRowCount = rs.getInt(1);
        assertEquals(0,dataTableRowCount);

        List<Future<?>> futureList = Lists.newArrayListWithExpectedSize(numThreads);
        for (int i =0; i<numThreads; ++i) {
            futureList.add(executorService.submit(new UpsertRunnable(fullTableName)));
        }
        // upsert some rows before creating the index 
        Thread.sleep(100);

        // create the index 
        try (Connection conn2 = DriverManager.getConnection(getUrl(), props)) {
            conn2.createStatement().execute(indexDDL);
        }

        // upsert some rows after creating the index
        Thread.sleep(50);
        // cancel the running threads
        stopThreads = true;
        executorService.shutdown();
        assertTrue(executorService.awaitTermination(30, TimeUnit.SECONDS));

        rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ COUNT(*) FROM " + fullTableName);
        assertTrue(rs.next());
        dataTableRowCount = rs.getInt(1);
        rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullIndexName);
        assertTrue(rs.next());
        int indexTableRowCount = rs.getInt(1);
        assertEquals("Data and Index table should have the same number of rows ", dataTableRowCount, indexTableRowCount);
    } finally {
        executorService.shutdownNow();
    }
}
 

@Test
public void testAlterAppendOnlySchema() throws Exception {
    try (Connection conn = DriverManager.getConnection(getUrl());
            Connection viewConn = isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL1) : conn ) {  
        String baseTableName = SchemaUtil.getTableName(SCHEMA1, generateUniqueName());
        String viewOfTable = SchemaUtil.getTableName(SCHEMA2, generateUniqueName());
        
        String ddl = "CREATE TABLE " + baseTableName + " (\n"
                +"%s ID VARCHAR(15) NOT NULL,\n"
                + " COL1 integer NOT NULL,"
                +"CREATED_DATE DATE,\n"
                +"CONSTRAINT PK PRIMARY KEY (%s ID, COL1)) %s";
        conn.createStatement().execute(generateDDL(ddl));
        ddl = "CREATE VIEW " + viewOfTable + " AS SELECT * FROM " + baseTableName;
        viewConn.createStatement().execute(ddl);
        
        PhoenixConnection phoenixConn = conn.unwrap(PhoenixConnection.class);
        PTable table = phoenixConn.getTable(new PTableKey(null, baseTableName));
        PName tenantId = isMultiTenant ? PNameFactory.newName(TENANT1) : null;
        assertFalse(table.isAppendOnlySchema());
        PTable viewTable = viewConn.unwrap(PhoenixConnection.class).getTable(new PTableKey(tenantId, viewOfTable));
        assertFalse(viewTable.isAppendOnlySchema());
        
        try {
            viewConn.createStatement().execute("ALTER VIEW " + viewOfTable + " SET APPEND_ONLY_SCHEMA = true");
            fail();
        }
        catch(SQLException e){
            assertEquals(SQLExceptionCode.CANNOT_ALTER_TABLE_PROPERTY_ON_VIEW.getErrorCode(), e.getErrorCode());
        }
        
        conn.createStatement().execute("ALTER TABLE " + baseTableName + " SET APPEND_ONLY_SCHEMA = true");
        viewConn.createStatement().execute("SELECT * FROM "+viewOfTable);
        
        phoenixConn = conn.unwrap(PhoenixConnection.class);
        table = phoenixConn.getTable(new PTableKey(null, baseTableName));
        assertTrue(table.isAppendOnlySchema());
        viewTable = viewConn.unwrap(PhoenixConnection.class).getTable(new PTableKey(tenantId, viewOfTable));
        assertTrue(viewTable.isAppendOnlySchema());
    }
}
 

@Test
public void testUpsertingNullForIndexedColumns() throws Exception {
    String tableName = "TBL_" + generateUniqueName();
    String indexName = "IDX_" + generateUniqueName();
    String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
    String testTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName + "_" + System.currentTimeMillis());
    try (Connection conn = getConnection()) {
        conn.setAutoCommit(false);
        ResultSet rs;
        Statement stmt = conn.createStatement();
        stmt.execute("CREATE TABLE " + testTableName + "(v1 VARCHAR PRIMARY KEY, v2 DOUBLE, v3 VARCHAR) "+tableDDLOptions);
        stmt.execute("CREATE " + (localIndex ? "LOCAL" : "") + " INDEX " + indexName + " ON " + testTableName + "  (v2) INCLUDE(v3)");
        
        //create a row with value null for indexed column v2
        stmt.executeUpdate("upsert into " + testTableName + " values('cc1', null, 'abc')");
        conn.commit();
        
        //assert values in index table 
        rs = stmt.executeQuery("select * from " + fullIndexName);
        assertTrue(rs.next());
        assertEquals(0, Doubles.compare(0, rs.getDouble(1)));
        assertTrue(rs.wasNull());
        assertEquals("cc1", rs.getString(2));
        assertEquals("abc", rs.getString(3));
        assertFalse(rs.next());
        
        //assert values in data table
        rs = stmt.executeQuery("select v1, v2, v3 from " + testTableName);
        assertTrue(rs.next());
        assertEquals("cc1", rs.getString(1));
        assertEquals(0, Doubles.compare(0, rs.getDouble(2)));
        assertTrue(rs.wasNull());
        assertEquals("abc", rs.getString(3));
        assertFalse(rs.next());
        
        //update the previously null value for indexed column v2 to a non-null value 1.23
        stmt.executeUpdate("upsert into " + testTableName + " values('cc1', 1.23, 'abc')");
        conn.commit();
        
        //assert values in data table
        rs = stmt.executeQuery("select /*+ NO_INDEX */ v1, v2, v3 from " + testTableName);
        assertTrue(rs.next());
        assertEquals("cc1", rs.getString(1));
        assertEquals(0, Doubles.compare(1.23, rs.getDouble(2)));
        assertEquals("abc", rs.getString(3));
        assertFalse(rs.next());
        
        //assert values in index table 
        rs = stmt.executeQuery("select * from " + fullIndexName);
        assertTrue(rs.next());
        assertEquals(0, Doubles.compare(1.23, rs.getDouble(1)));
        assertEquals("cc1", rs.getString(2));
        assertEquals("abc", rs.getString(3));
        assertFalse(rs.next());
        
        //update the value for indexed column v2 back to null
        stmt.executeUpdate("upsert into " + testTableName + " values('cc1', null, 'abc')");
        conn.commit();
        
        //assert values in index table 
        rs = stmt.executeQuery("select * from " + fullIndexName);
        assertTrue(rs.next());
        assertEquals(0, Doubles.compare(0, rs.getDouble(1)));
        assertTrue(rs.wasNull());
        assertEquals("cc1", rs.getString(2));
        assertEquals("abc", rs.getString(3));
        assertFalse(rs.next());
        
        //assert values in data table
        rs = stmt.executeQuery("select v1, v2, v3 from " + testTableName);
        assertTrue(rs.next());
        assertEquals("cc1", rs.getString(1));
        assertEquals(0, Doubles.compare(0, rs.getDouble(2)));
        assertEquals("abc", rs.getString(3));
        assertFalse(rs.next());
    } 
}