org.apache.phoenix.query.ConnectionQueryServices Java Examples

private List<PeekingResultIterator> recreateIterators(ConnectionQueryServices services,
        boolean isLocalIndex, Queue<PeekingResultIterator> allIterators,
        List<PeekingResultIterator> iterators, boolean isReverse, long maxQueryEndTime,
        ScanWrapper previousScan, boolean clearedCache,
        List<PeekingResultIterator> concatIterators,
        Iterator<Pair<Scan, Future<PeekingResultIterator>>> scanPairItr,
        Pair<Scan, Future<PeekingResultIterator>> scanPair, int retryCount) throws SQLException {
    scanPairItr.remove();
    // Resubmit just this portion of work again
    Scan oldScan = scanPair.getFirst();
    byte[] startKey = oldScan.getAttribute(SCAN_ACTUAL_START_ROW);
    byte[] endKey = oldScan.getStopRow();

    List<List<Scan>> newNestedScans = this.getParallelScans(startKey, endKey);
    // Add any concatIterators that were successful so far
    // as we need these to be in order
    addIterator(iterators, concatIterators);
    concatIterators = Lists.newArrayList();
    getIterators(newNestedScans, services, isLocalIndex, allIterators, iterators, isReverse,
            maxQueryEndTime, newNestedScans.size(), previousScan, retryCount);
    return concatIterators;
}
 

private long executeToolForImmutableTables(ConnectionQueryServices queryServices,
        List<String> immutableList) {
    if (immutableList.isEmpty()) {
        return 0;
    }
    LOGGER.info("Started " + operation + " for immutable tables");
    List<String> failedTables = new ArrayList<String>();
    for (String dataTableFullName : immutableList) {
        try (Admin admin = queryServices.getAdmin()) {
            HashSet<String> indexes = tablesAndIndexes.get(dataTableFullName);
            LOGGER.info("Executing " + operation + " of " + dataTableFullName
                    + " (immutable)");
            disableTable(admin, dataTableFullName, indexes);
            modifyTable(admin, dataTableFullName, indexes);
        } catch (Throwable e) {
            LOGGER.severe("Something went wrong while disabling "
                    + "or modifying immutable table " + e);
            handleFailure(queryServices, dataTableFullName, immutableList, failedTables);
        }
    }
    immutableList.removeAll(failedTables);
    long startWaitTime = EnvironmentEdgeManager.currentTimeMillis();
    return startWaitTime;
}
 

/**
 * @return Pair of numbers in which the first part is estimated number of bytes that will be
 *         scanned and the second part is estimated number of rows. Returned value is null if
 *         estimated size of data to scan is beyond a threshold.
 * @throws SQLException
 */
private static Pair<Long, Long> getEstimateOfDataSizeToScanIfWithinThreshold(StatementContext context, PTable table, Integer perScanLimit) throws SQLException {
    Scan scan = context.getScan();
    ConnectionQueryServices services = context.getConnection().getQueryServices();
    long estRowSize = SchemaUtil.estimateRowSize(table);
    long regionSize = services.getProps().getLong(HConstants.HREGION_MAX_FILESIZE,
            HConstants.DEFAULT_MAX_FILE_SIZE);
    if (perScanLimit == null || scan.getFilter() != null) {
        /*
         * If a limit is not provided or if we have a filter, then we are not able to decide whether
         * the amount of data we need to scan is less than the threshold.
         */
        return null;
    } 
    float factor =
        services.getProps().getFloat(QueryServices.LIMITED_QUERY_SERIAL_THRESHOLD,
            QueryServicesOptions.DEFAULT_LIMITED_QUERY_SERIAL_THRESHOLD);
    long threshold = (long)(factor * regionSize);
    long estimatedBytes = perScanLimit * estRowSize;
    long estimatedRows = perScanLimit;
    return (perScanLimit * estRowSize < threshold) ? new Pair<>(estimatedBytes, estimatedRows) : null;
}
 

@Test
public void testUpgradeOnlyHappensOnce() throws Exception {
    ConnectionQueryServices services = DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class).getQueryServices();
    assertTrue(services instanceof PhoenixUpgradeCountingServices);
    // Check if the timestamp version is changing between the current version and prior version
    boolean wasTimestampChanged = systemTableVersion != MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP;
    reinitialize = true;
    systemTableVersion = MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP;
    DriverManager.getConnection(getUrl());
    // Confirm that if the timestamp changed, that an upgrade was performed (and that if it
    // didn't, that an upgrade wasn't attempted).
    assertEquals(wasTimestampChanged ? 1 : 0, countUpgradeAttempts);
    // Confirm that another connection does not increase the number of times upgrade was attempted
    DriverManager.getConnection(getUrl());
    assertEquals(wasTimestampChanged ? 1 : 0, countUpgradeAttempts);
}
 

protected void splitTable(Connection conn, byte[] splitPoint, byte[] tabName) throws IOException, InterruptedException, SQLException {
    ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices();
    int nRegionsNow = services.getAllTableRegions(tabName).size();
    HBaseAdmin admin = services.getAdmin();
    try {
        admin.split(tabName, splitPoint);
        int nTries = 0;
        int nRegions;
        do {
            Thread.sleep(2000);
            services.clearTableRegionCache(tabName);
            nRegions = services.getAllTableRegions(tabName).size();
            nTries++;
        } while (nRegions == nRegionsNow && nTries < 10);
        if (nRegions == nRegionsNow) {
            fail();
        }
        // FIXME: I see the commit of the stats finishing before this with a lower timestamp that the scan timestamp,
        // yet without this sleep, the query finds the old data. Seems like an HBase bug and a potentially serious one.
        Thread.sleep(8000);
    } finally {
        admin.close();
    }
}
 

/**
 * Executes the scan in parallel across all regions, blocking until all scans are complete.
 * @return the result iterators for the scan of each region
 */
@Override
public List<PeekingResultIterator> getIterators() throws SQLException {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug(LogUtil.addCustomAnnotations("Getting iterators for " + this,
                ScanUtil.getCustomAnnotations(scan)) + "on table " + context.getCurrentTable().getTable().getName());
    }
    boolean isReverse = ScanUtil.isReversed(scan);
    boolean isLocalIndex = getTable().getIndexType() == IndexType.LOCAL;
    final ConnectionQueryServices services = context.getConnection().getQueryServices();
    // Get query time out from Statement
    final long startTime = EnvironmentEdgeManager.currentTimeMillis();
    final long maxQueryEndTime = startTime + context.getStatement().getQueryTimeoutInMillis();
    int numScans = size();
    // Capture all iterators so that if something goes wrong, we close them all
    // The iterators list is based on the submission of work, so it may not
    // contain them all (for example if work was rejected from the queue)
    Queue<PeekingResultIterator> allIterators = new ConcurrentLinkedQueue<>();
    List<PeekingResultIterator> iterators = new ArrayList<PeekingResultIterator>(numScans);
    ScanWrapper previousScan = new ScanWrapper(null);
    return getIterators(scans, services, isLocalIndex, allIterators, iterators, isReverse, maxQueryEndTime,
            splits.size(), previousScan, context.getConnection().getQueryServices().getConfiguration()
                    .getInt(QueryConstants.HASH_JOIN_CACHE_RETRIES, QueryConstants.DEFAULT_HASH_JOIN_CACHE_RETRIES));
}
 

private static int getRowCountForEmptyColValue(Connection conn, String tableName,
        byte[] valueBytes)  throws IOException, SQLException {

    PTable table = PhoenixRuntime.getTable(conn, tableName);
    byte[] emptyCF = SchemaUtil.getEmptyColumnFamily(table);
    byte[] emptyCQ = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst();
    ConnectionQueryServices queryServices =
            conn.unwrap(PhoenixConnection.class).getQueryServices();
    HTable htable = (HTable) queryServices.getTable(table.getPhysicalName().getBytes());
    Scan scan = new Scan();
    scan.addColumn(emptyCF, emptyCQ);
    ResultScanner resultScanner = htable.getScanner(scan);
    int count = 0;

    for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
        if (Bytes.compareTo(result.getValue(emptyCF, emptyCQ), 0, valueBytes.length,
                valueBytes, 0, valueBytes.length) == 0) {
            ++count;
        }
    }
    return count;
}
 

@Test
/**
 * Test delete before rebuild
 */
public void testDeleteBeforeRebuildForGlobalIndex() throws Exception {
    conn.createStatement().execute(String.format(INDEX_GLOBAL_DDL, globalIndexName, dataTableFullName));
    String globalIndexUpsert = String.format(UPSERT_SQL, globalIndexFullName);
    PreparedStatement stmt = conn.prepareStatement(globalIndexUpsert);
    upsertRow(stmt, "tenantID1",11, "name11", 99911);
    conn.commit();

    ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices();
    PTable physicalTable = PhoenixRuntime.getTable(conn, globalIndexFullName);
    Table hIndexTable= queryServices.getTable(physicalTable.getPhysicalName().getBytes());
    int count = getUtility().countRows(hIndexTable);
    // Confirm index has rows.
    assertEquals(4, count);

    runIndexTool(schemaName, dataTableName, globalIndexName, 0);

    count = getUtility().countRows(hIndexTable);

    // Confirm index has all the data rows
    assertEquals(3, count);
}
 

@Test
public void testSetRPCTimeOnConnection() throws Exception {
    Properties overriddenProps = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    overriddenProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
    overriddenProps.setProperty("hbase.rpc.timeout", Long.toString(100));
    String url = QueryUtil.getConnectionUrl(overriddenProps, config, "longRunning");
    Connection conn1 = DriverManager.getConnection(url, overriddenProps);
    ConnectionQueryServices s1 = conn1.unwrap(PhoenixConnection.class).getQueryServices();
    ReadOnlyProps configProps = s1.getProps();
    assertEquals("100", configProps.get("hbase.rpc.timeout"));
    
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    props.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
    Connection conn2 = DriverManager.getConnection(getUrl(), props);
    ConnectionQueryServices s2 = conn2.unwrap(PhoenixConnection.class).getQueryServices();
    assertFalse(s1 == s2);
    Connection conn3 = DriverManager.getConnection(getUrl(), props);
    ConnectionQueryServices s3 = conn3.unwrap(PhoenixConnection.class).getQueryServices();
    assertTrue(s2 == s3);
    
    Connection conn4 = DriverManager.getConnection(url, overriddenProps);
    ConnectionQueryServices s4 = conn4.unwrap(PhoenixConnection.class).getQueryServices();
    assertTrue(s1 == s4);
}
 

public ServerCache(byte[] id, Set<HRegionLocation> servers, ImmutableBytesWritable cachePtr,
        ConnectionQueryServices services, boolean storeCacheOnClient) throws IOException {
    maxServerCacheTTL = services.getProps().getInt(
            QueryServices.MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB,
            QueryServicesOptions.DEFAULT_MAX_SERVER_CACHE_TIME_TO_LIVE_MS);
    this.id = id;
    this.servers = new HashMap();
    long currentTime = EnvironmentEdgeManager.currentTimeMillis();
    for(HRegionLocation loc : servers) {
        this.servers.put(loc, currentTime);
    }
    this.size =  cachePtr.getLength();
    if (storeCacheOnClient) {
        try {
            this.chunk = services.getMemoryManager().allocate(cachePtr.getLength());
            this.cachePtr = cachePtr;
        } catch (InsufficientMemoryException e) {
            this.outputFile = File.createTempFile("HashJoinCacheSpooler", ".bin", new File(services.getProps()
                    .get(QueryServices.SPOOL_DIRECTORY, QueryServicesOptions.DEFAULT_SPOOL_DIRECTORY)));
            try (OutputStream fio = Files.newOutputStream(outputFile.toPath())) {
                fio.write(cachePtr.get(), cachePtr.getOffset(), cachePtr.getLength());
            }
        }
    }
    
}
 

@Test
public void testAcquiringAndReleasingUpgradeMutex() throws Exception {
    ConnectionQueryServices services = null;
    try (Connection conn = getConnection(false, null)) {
        services = conn.unwrap(PhoenixConnection.class).getQueryServices();
        assertTrue(((ConnectionQueryServicesImpl)services)
                .acquireUpgradeMutex(MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0));
        try {
            ((ConnectionQueryServicesImpl)services)
                    .acquireUpgradeMutex(MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
            fail();
        } catch (UpgradeInProgressException expected) {

        }
        ((ConnectionQueryServicesImpl)services).releaseUpgradeMutex();
    }
}
 

@Test
public void testEmptyGuidePostGeneratedWhenDataSizeLessThanGPWidth() throws Exception {
    try (Connection conn = getConnection()) {
        long guidePostWidth = 20000000;
        conn.createStatement()
                .execute("CREATE TABLE " + fullTableName
                        + " ( k INTEGER, c1.a bigint,c2.b bigint CONSTRAINT pk PRIMARY KEY (k)) GUIDE_POSTS_WIDTH="
                        + guidePostWidth + ", SALT_BUCKETS = 4");
        conn.createStatement().execute("upsert into " + fullTableName + " values (100,1,3)");
        conn.createStatement().execute("upsert into " + fullTableName + " values (101,2,4)");
        conn.commit();
        collectStatistics(conn, fullTableName);
        ConnectionQueryServices queryServices =
                conn.unwrap(PhoenixConnection.class).getQueryServices();
        verifyGuidePostGenerated(queryServices, physicalTableName, new String[] {"C1", "C2"}, guidePostWidth, true);
    }
}
 

private void verifyGuidePostGenerated(ConnectionQueryServices queryServices,
        String tableName, String[] familyNames,
        long guidePostWidth, boolean emptyGuidePostExpected) throws Exception {
    try (Table statsHTable =
            queryServices.getTable(
                    SchemaUtil.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES,
                            queryServices.getProps()).getName())) {
        for (String familyName : familyNames) {
            GuidePostsInfo gps =
                    StatisticsUtil.readStatistics(statsHTable,
                            new GuidePostsKey(Bytes.toBytes(tableName), Bytes.toBytes(familyName)),
                            HConstants.LATEST_TIMESTAMP);
            assertTrue(emptyGuidePostExpected ? gps.isEmptyGuidePost() : !gps.isEmptyGuidePost());
            assertTrue(gps.getByteCounts()[0] >= guidePostWidth);
            assertTrue(gps.getGuidePostTimestamps()[0] > 0);
        }
    }
}
 

public boolean addServerCache(byte[] startkeyOfRegion, ServerCache cache, HashCacheFactory cacheFactory,
          byte[] txState, PTable pTable) throws Exception {
     Table table = null;
     boolean success = true;
     byte[] cacheId = cache.getId();
     try {
         ConnectionQueryServices services = connection.getQueryServices();
         
         byte[] tableName = pTable.getPhysicalName().getBytes();
         table = services.getTable(tableName);
         HRegionLocation tableRegionLocation = services.getTableRegionLocation(tableName, startkeyOfRegion);
         if(cache.isExpired(tableRegionLocation)) {
             return false;
         }
if (cache.addServer(tableRegionLocation) || services.getProps().getBoolean(HASH_JOIN_SERVER_CACHE_RESEND_PER_SERVER,false)) {
	success = addServerCache(table, startkeyOfRegion, pTable, cacheId, cache.getCachePtr(), cacheFactory,
			txState, false);
}
return success;
     } finally {
         Closeables.closeQuietly(table);
     }
 }
 

private void deleteBeforeRebuild(Connection conn) throws SQLException, IOException {
    if (MetaDataUtil.isViewIndex(pIndexTable.getPhysicalName().getString())) {
        throw new IllegalArgumentException(String.format(
            "%s is a view index. delete-all-and-rebuild is not supported for view indexes",
            indexTable));
    }

    if (isLocalIndexBuild) {
        throw new IllegalArgumentException(String.format(
                "%s is a local index.  delete-all-and-rebuild is not supported for local indexes", indexTable));
    } else {
        ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices();
        try (Admin admin = queryServices.getAdmin()){
            TableName tableName = TableName.valueOf(qIndexTable);
            admin.disableTable(tableName);
            admin.truncateTable(tableName, true);
        }
    }
}
 

@VisibleForTesting
public IndexVerificationOutputRepository(byte[] indexName, Connection conn) throws SQLException {
    ConnectionQueryServices queryServices =
        conn.unwrap(PhoenixConnection.class).getQueryServices();
    outputTable = queryServices.getTable(OUTPUT_TABLE_NAME_BYTES);
    indexTable = queryServices.getTable(indexName);
}
 

@Test
public void testRoundRobinAfterTableSplit() throws Exception {
    String tableName = generateUniqueName();
    byte[] tableNameBytes = Bytes.toBytes(tableName);
    int numRows = setupTableForSplit(tableName);
    Connection conn = getConnection();
    ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices();
    int nRegions = services.getAllTableRegions(tableNameBytes).size();
    int nRegionsBeforeSplit = nRegions;
    Admin admin = services.getAdmin();
    try {
        // Split is an async operation. So hoping 10 seconds is long enough time.
        // If the test tends to flap, then you might want to increase the wait time
        admin.split(TableName.valueOf(tableName));
        CountDownLatch latch = new CountDownLatch(1);
        int nTries = 0;
        long waitTimeMillis = 2000;
        while (nRegions == nRegionsBeforeSplit && nTries < 10) {
            latch.await(waitTimeMillis, TimeUnit.MILLISECONDS);
            nRegions = services.getAllTableRegions(tableNameBytes).size();
            nTries++;
        }
        
        String query = "SELECT * FROM " + tableName;
        Statement stmt = conn.createStatement();
        stmt.setFetchSize(10); // this makes scanner caches to be replenished in parallel.
        ResultSet rs = stmt.executeQuery(query);
        int numRowsRead = 0;
        while (rs.next()) {
            numRowsRead++;
        }
        nRegions = services.getAllTableRegions(tableNameBytes).size();
        // Region cache has been updated, as there are more regions now
        assertNotEquals(nRegions, nRegionsBeforeSplit);
        assertEquals(numRows, numRowsRead);
    } finally {
        admin.close();
    }

}
 

long getNumCachedConnections(Connection conn) throws Exception {
  PhoenixConnection pConn = conn.unwrap(PhoenixConnection.class);
  ConnectionQueryServices cqs = pConn.getQueryServices();
  // For whatever reason, we sometimes get a delegate here, and sometimes the real thing.
  if (cqs instanceof DelegateConnectionQueryServices) {
    cqs = ((DelegateConnectionQueryServices) cqs).getDelegate();
  }
  assertTrue("ConnectionQueryServices was a " + cqs.getClass(), cqs instanceof ConnectionQueryServicesImpl);
  ConnectionQueryServicesImpl cqsi = (ConnectionQueryServicesImpl) cqs;
  long cachedConnections = 0L;
  for (LinkedBlockingQueue<WeakReference<PhoenixConnection>> queue : cqsi.getCachedConnections()) {
    cachedConnections += queue.size();
  }
  return cachedConnections;
}
 

@Test
public void testConcurrentUpgradeThrowsUprgadeInProgressException() throws Exception {
    final AtomicBoolean mutexStatus1 = new AtomicBoolean(false);
    final AtomicBoolean mutexStatus2 = new AtomicBoolean(false);
    final CountDownLatch latch = new CountDownLatch(2);
    final AtomicInteger numExceptions = new AtomicInteger(0);
    ConnectionQueryServices services = null;
    final byte[] mutexKey = Bytes.toBytes(generateUniqueName());
    try (Connection conn = getConnection(false, null)) {
        services = conn.unwrap(PhoenixConnection.class).getQueryServices();
        FutureTask<Void> task1 = new FutureTask<>(new AcquireMutexRunnable(mutexStatus1, services, latch, numExceptions, mutexKey));
        FutureTask<Void> task2 = new FutureTask<>(new AcquireMutexRunnable(mutexStatus2, services, latch, numExceptions, mutexKey));
        Thread t1 = new Thread(task1);
        t1.setDaemon(true);
        Thread t2 = new Thread(task2);
        t2.setDaemon(true);
        t1.start();
        t2.start();
        latch.await();
        // make sure tasks didn't fail by calling get()
        task1.get();
        task2.get();
        assertTrue("One of the threads should have acquired the mutex", mutexStatus1.get() || mutexStatus2.get());
        assertNotEquals("One and only one thread should have acquired the mutex ", mutexStatus1.get(),
                mutexStatus2.get());
        assertEquals("One and only one thread should have caught UpgradeRequiredException ", 1, numExceptions.get());
    }
}
 

@Test
public void testUpgradingConnectionBypassesUpgradeRequiredCheck() throws Exception {
    String tableName = generateUniqueName();
    try (Connection conn = getConnection(false, null)) {
        conn.createStatement()
                .execute(
                        "CREATE TABLE "
                                + tableName
                                + " (PK1 VARCHAR NOT NULL, PK2 VARCHAR, KV1 VARCHAR, KV2 VARCHAR CONSTRAINT PK PRIMARY KEY(PK1, PK2))");
        final ConnectionQueryServices delegate = conn.unwrap(PhoenixConnection.class).getQueryServices();
        ConnectionQueryServices servicesWithUpgrade = new DelegateConnectionQueryServices(delegate) {
            @Override
            public boolean isUpgradeRequired() {
                return true;
            }
        };
        try (PhoenixConnection phxConn = new PhoenixConnection(conn.unwrap(PhoenixConnection.class),
                servicesWithUpgrade, conn.getClientInfo())) {
            // Because upgrade is required, this SQL should fail.
            try {
                phxConn.createStatement().executeQuery("SELECT * FROM " + tableName);
                fail("SELECT should have failed with UpgradeRequiredException");
            } catch (UpgradeRequiredException expected) {

            }
            // Marking connection as the one running upgrade should let SQL execute fine.
            phxConn.setRunningUpgrade(true);
            phxConn.createStatement().execute(
                    "UPSERT INTO " + tableName + " VALUES ('PK1', 'PK2', 'KV1', 'KV2')" );
            phxConn.commit();
            try (ResultSet rs = phxConn.createStatement().executeQuery("SELECT * FROM " + tableName)) {
                assertTrue(rs.next());
                assertFalse(rs.next());
            }
        }
    }
}
 

private void changeMutexLock(Properties clientProps, boolean acquire) throws SQLException, IOException {
    ConnectionQueryServices services = null;

    try (Connection conn = DriverManager.getConnection(getJdbcUrl(), clientProps)) {
        services = conn.unwrap(PhoenixConnection.class).getQueryServices();
        if(acquire) {
           assertTrue(((ConnectionQueryServicesImpl) services)
                    .acquireUpgradeMutex(MetaDataProtocol.MIN_SYSTEM_TABLE_MIGRATION_TIMESTAMP));
        } else {
            services.deleteMutexCell(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA,
                PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE, null, null);
        }
    }
}
 

public AcquireMutexRunnable(AtomicBoolean acquireStatus, ConnectionQueryServices services, CountDownLatch latch, AtomicInteger numExceptions, byte[] mutexKey) {
    this.acquireStatus = acquireStatus;
    this.services = services;
    this.latch = latch;
    this.numExceptions = numExceptions;
    this.mutexRowKey = mutexKey;
}
 

private void testWithOutput(OutputFormat outputFormat) throws Exception {
    connTenant.createStatement()
            .execute(String.format(upsertQueryStr, tenantViewName, tenantId, 1, "x"));
    connTenant.createStatement()
            .execute(String.format(upsertQueryStr, tenantViewName, tenantId, 2, "x2"));
    connTenant.createStatement()
            .execute(String.format(upsertQueryStr, tenantViewName, tenantId, 3, "x3"));
    connTenant.createStatement().execute(String.format("UPSERT INTO %s (\":ID\", \"0:NAME\") values (%d, '%s')",
            indexNameTenant, 5555, "wrongName"));
    connTenant.commit();

    ConnectionQueryServices queryServices = connGlobal.unwrap(PhoenixConnection.class).getQueryServices();
    Admin admin = queryServices.getAdmin();
    TableName tableName = TableName.valueOf(viewIndexTableName);
    admin.disableTable(tableName);
    admin.truncateTable(tableName, false);

    String[]
            argValues =
            getArgValues("", tenantViewName, indexNameTenant, 10L, SourceTable.DATA_TABLE_SOURCE, true, outputFormat, null,
                    tenantId, EnvironmentEdgeManager.currentTimeMillis());
    List<Job> completedJobs = runScrutiny(argValues);

    assertEquals(1, completedJobs.size());
    for (Job job : completedJobs) {
        assertTrue(job.isSuccessful());
        Counters counters = job.getCounters();
        assertEquals(0, getCounterValue(counters, VALID_ROW_COUNT));
        assertEquals(3, getCounterValue(counters, INVALID_ROW_COUNT));
    }
}
 

private int executeTool(Connection conn,
        ConnectionQueryServices queryServices,
        Configuration conf) {
    ArrayList<String> immutableList = new ArrayList<>();
    ArrayList<String> mutableList = new ArrayList<>();
    for (Map.Entry<String, HashSet<String>> entry :tablesAndIndexes.entrySet()) {
        String dataTableFullName = entry.getKey();
        try {
            PTable dataTable = PhoenixRuntime.getTableNoCache(conn, dataTableFullName);
            if (dataTable.isImmutableRows()) {
                //add to list where immutable tables are processed in a different function
                immutableList.add(dataTableFullName);
            } else {
                mutableList.add(dataTableFullName);
            }
        } catch (SQLException e) {
            LOGGER.severe("Something went wrong while getting the PTable "
                    + dataTableFullName + " " + e);
            return -1;
        }
    }
    long startWaitTime = executeToolForImmutableTables(queryServices, immutableList);
    executeToolForMutableTables(conn, queryServices, conf, mutableList);
    enableImmutableTables(queryServices, immutableList, startWaitTime);
    rebuildIndexes(conn, conf, immutableList);
    if (hasFailure) {
        return -1;
    } else {
        return 0;
    }
}
 

@Test
public void testCollectingAllVersionsOfCells() throws Exception {
    try (Connection conn = getConnection()) {
        long guidePostWidth = 70;
        String ddl =
                "CREATE TABLE " + fullTableName + " (k INTEGER PRIMARY KEY, c1.a bigint, c2.b bigint)"
                        + " GUIDE_POSTS_WIDTH=" + guidePostWidth
                        + ", USE_STATS_FOR_PARALLELIZATION=true" + ", VERSIONS=3";
        conn.createStatement().execute(ddl);
        conn.createStatement().execute("upsert into " + fullTableName + " values (100,100,3)");
        conn.commit();
        collectStatistics(conn, fullTableName);

        ConnectionQueryServices queryServices =
                conn.unwrap(PhoenixConnection.class).getQueryServices();

        // The table only has one row. All cells just has one version, and the data size of the row
        // is less than the guide post width, so we generate empty guide post.
        verifyGuidePostGenerated(queryServices, physicalTableName, new String[] {"C1", "C2"}, guidePostWidth, true);

        conn.createStatement().execute("upsert into " + fullTableName + " values (100,101,4)");
        conn.commit();
        collectStatistics(conn, fullTableName);

        // We updated the row. Now each cell has two versions, and the data size of the row
        // is >= the guide post width, so we generate non-empty guide post.
        verifyGuidePostGenerated(queryServices, physicalTableName, new String[] {"C1", "C2"}, guidePostWidth, false);
    }
}
 

public void createResultTable(Connection connection) throws IOException, SQLException {
    ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices();
    Admin admin = queryServices.getAdmin();
    TableName resultTableName = TableName.valueOf(RESULT_TABLE_NAME);
    if (!admin.tableExists(resultTableName)) {
        ColumnFamilyDescriptor columnDescriptor =
            ColumnFamilyDescriptorBuilder.newBuilder(RESULT_TABLE_COLUMN_FAMILY).
                setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL).build();
        TableDescriptor tableDescriptor =
            TableDescriptorBuilder.newBuilder(resultTableName).
                setColumnFamily(columnDescriptor).build();
        admin.createTable(tableDescriptor);
        resultHTable = admin.getConnection().getTable(resultTableName);
    }
}
 

public void createOutputTable(Connection connection) throws IOException, SQLException {
    ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices();
    Admin admin = queryServices.getAdmin();
    TableName outputTableName = TableName.valueOf(OUTPUT_TABLE_NAME);
    if (!admin.tableExists(outputTableName)) {
        ColumnFamilyDescriptor columnDescriptor =
            ColumnFamilyDescriptorBuilder.newBuilder(OUTPUT_TABLE_COLUMN_FAMILY).
                setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL).build();
        TableDescriptor tableDescriptor =
            TableDescriptorBuilder.newBuilder(outputTableName).
                setColumnFamily(columnDescriptor).build();
        admin.createTable(tableDescriptor);
        outputTable = admin.getConnection().getTable(outputTableName);
    }
}
 

/**
 * Synchronize certain properties across column families of global index tables for a given base table
 * @param cqs CQS object to get table descriptor from PTable
 * @param baseTable base table
 * @param defaultColFam column family to be used for synchronizing properties
 * @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor
 * @param tableDescsToSync set of modified table descriptors
 */
private static void syncGlobalIndexesForTable(ConnectionQueryServices cqs, PTable baseTable, ColumnFamilyDescriptor defaultColFam,
        Map<String, Object> syncedProps, Set<TableDescriptor> tableDescsToSync) throws SQLException {
    for (PTable indexTable: baseTable.getIndexes()) {
        // We already handle local index property synchronization when considering all column families of the base table
        if (indexTable.getIndexType() == IndexType.GLOBAL) {
            addTableDescIfPropsChanged(cqs.getTableDescriptor(indexTable.getPhysicalName().getBytes()),
                    defaultColFam, syncedProps, tableDescsToSync);
        }
    }
}
 

private void incrementSequenceValues() throws SQLException {
    if (sequenceMap == null) {
        return;
    }
    Long scn = statement.getConnection().getSCN();
    long timestamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
    ConnectionQueryServices services = statement.getConnection().getQueryServices();
    services.incrementSequences(nextSequences, timestamp, srcSequenceValues, sqlExceptions);
    setSequenceValues(srcSequenceValues, dstSequenceValues, sqlExceptions);
    int offset = nextSequences.size();
    for (int i = 0; i < currentSequences.size(); i++) {
        dstSequenceValues[sequencePosition[offset+i]] = services.currentSequenceValue(currentSequences.get(i), timestamp);
    }
}
 

@Override // public for testing
public synchronized ConnectionQueryServices getConnectionQueryServices(String url, Properties info) throws SQLException {
    checkClosed();
    if (connectionQueryServices != null) { return connectionQueryServices; }
    ConnectionInfo connInfo = ConnectionInfo.create(url);
    if (connInfo.isConnectionless()) {
        connectionQueryServices = new ConnectionlessQueryServicesImpl(queryServices, connInfo, info);
    } else {
        connectionQueryServices = new ConnectionQueryServicesTestImpl(queryServices, connInfo, info);
    }
    connectionQueryServices.init(url, info);
    return connectionQueryServices;
}