Java Code Examples for org.apache.hadoop.hbase.client.Mutation#getRow()

@Override
public MetaDataMutationResult createTable(final List<Mutation> tableMetaData, byte[] tableName, PTableType tableType,
        Map<String,Object> tableProps, final List<Pair<byte[],Map<String,Object>>> families, byte[][] splits) throws SQLException {
    byte[][] rowKeyMetadata = new byte[3][];
    Mutation m = tableMetaData.get(0);
    byte[] key = m.getRow();
    SchemaUtil.getVarChars(key, rowKeyMetadata);
    byte[] tenantIdBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
    byte[] schemaBytes = rowKeyMetadata[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
    byte[] tableBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
    if (tableType != PTableType.VIEW || tableName != null) {
        tableName = tableName == null ? SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes) : tableName;
        ensureTableCreated(tableName, tableType, tableProps, families, splits);
    }
    
    byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaBytes, tableBytes);
    MetaDataMutationResult result = metaDataCoprocessorExec(tableKey,
        new Batch.Call<MetaDataProtocol, MetaDataMutationResult>() {
            @Override
            public MetaDataMutationResult call(MetaDataProtocol instance) throws IOException {
              return instance.createTable(tableMetaData);
            }
        });
    return result;
}
 

private static Get toGet(final Mutation mutation) throws IOException {
  if(!(mutation instanceof Increment) && !(mutation instanceof Append)) {
    throw new AssertionError("mutation must be a instance of Increment or Append");
  }
  Get get = new Get(mutation.getRow());
  CellScanner cellScanner = mutation.cellScanner();
  while (!cellScanner.advance()) {
    Cell cell = cellScanner.current();
    get.addColumn(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell));
  }
  if (mutation instanceof Increment) {
    // Increment
    Increment increment = (Increment) mutation;
    get.setTimeRange(increment.getTimeRange().getMin(), increment.getTimeRange().getMax());
  } else {
    // Append
    Append append = (Append) mutation;
    get.setTimeRange(append.getTimeRange().getMin(), append.getTimeRange().getMax());
  }
  for (Entry<String, byte[]> entry : mutation.getAttributesMap().entrySet()) {
    get.setAttribute(entry.getKey(), entry.getValue());
  }
  return get;
}
 

private Collection<? extends Mutation> groupMutations(MiniBatchOperationInProgress<Mutation> miniBatchOp,
                                                      BatchMutateContext context) throws IOException {
    context.multiMutationMap = new HashMap<>();
    for (int i = 0; i < miniBatchOp.size(); i++) {
        Mutation m = miniBatchOp.getOperation(i);
        // skip this mutation if we aren't enabling indexing
        // unfortunately, we really should ask if the raw mutation (rather than the combined mutation)
        // should be indexed, which means we need to expose another method on the builder. Such is the
        // way optimization go though.
        if (miniBatchOp.getOperationStatus(i) != IGNORE && this.builder.isEnabled(m)) {
            ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow());
            MultiMutation stored = context.multiMutationMap.get(row);
            if (stored == null) {
                // we haven't seen this row before, so add it
                stored = new MultiMutation(row);
                context.multiMutationMap.put(row, stored);
            }
            stored.addAll(m);
        }
    }
    return context.multiMutationMap.values();
}
 

private Mutation getUnverifiedPutMutation(Mutation orig, Long ts) {
    Mutation m = new Put(orig.getRow());
    if (orig.getAttributesMap() != null) {
        for (Map.Entry<String,byte[]> entry : orig.getAttributesMap().entrySet()) {
            m.setAttribute(entry.getKey(), entry.getValue());
        }
    }
    List<Cell> origList = orig.getFamilyCellMap().firstEntry().getValue();
    ts = ts == null ? EnvironmentEdgeManager.currentTimeMillis() : ts;
    Cell c = getNewPutCell(orig, origList, ts, KeyValue.Type.Put);
    Cell empty = getEmptyCell(orig, origList, ts, KeyValue.Type.Put, false);
    byte[] fam = CellUtil.cloneFamily(origList.get(0));
    List<Cell> famCells = Lists.newArrayList();
    m.getFamilyCellMap().put(fam, famCells);
    famCells.add(c);
    famCells.add(empty);
    return m;
}
 

@VisibleForTesting
public int prepareIndexMutations(Put put, Delete del) throws IOException {
    List<Mutation> indexMutations = prepareIndexMutationsForRebuild(indexMaintainer, put, del);
    for (Mutation mutation : indexMutations) {
        byte[] indexRowKey = mutation.getRow();
        List<Mutation> mutationList = indexKeyToMutationMap.get(indexRowKey);
        if (mutationList == null) {
            mutationList = new ArrayList<>();
            mutationList.add(mutation);
            indexKeyToMutationMap.put(indexRowKey, mutationList);
        } else {
            mutationList.add(mutation);
        }
    }
    return 0;
}
 

private static byte[] getTableName(List<Mutation> tableMetaData, byte[] physicalTableName) {
    if (physicalTableName != null) {
        return physicalTableName;
    }
    byte[][] rowKeyMetadata = new byte[3][];
    Mutation m = MetaDataUtil.getTableHeaderRow(tableMetaData);
    byte[] key = m.getRow();
    SchemaUtil.getVarChars(key, rowKeyMetadata);
    byte[] schemaBytes = rowKeyMetadata[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
    byte[] tableBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
    return SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
}
 

private static byte[] getTableName(List<Mutation> tableMetaData, byte[] physicalTableName) {
    if (physicalTableName != null) {
        return physicalTableName;
    }
    byte[][] rowKeyMetadata = new byte[3][];
    Mutation m = MetaDataUtil.getTableHeaderRow(tableMetaData);
    byte[] key = m.getRow();
    SchemaUtil.getVarChars(key, rowKeyMetadata);
    byte[] schemaBytes = rowKeyMetadata[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
    byte[] tableBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
    return SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
}
 

private void populateRowsToLock(MiniBatchOperationInProgress<Mutation> miniBatchOp, BatchMutateContext context) {
    for (int i = 0; i < miniBatchOp.size(); i++) {
        if (miniBatchOp.getOperationStatus(i) == IGNORE) {
            continue;
        }
        Mutation m = miniBatchOp.getOperation(i);
        if (this.builder.isEnabled(m)) {
            ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow());
            if (!context.rowsToLock.contains(row)) {
                context.rowsToLock.add(row);
            }
        }
    }
}
 

/**
 * This method applies the pending put mutations on the the next row states.
 * Before this method is called, the next row states is set to current row states.
 */
private void applyPendingPutMutations(MiniBatchOperationInProgress<Mutation> miniBatchOp,
                                      BatchMutateContext context, long now) throws IOException {
    for (Integer i = 0; i < miniBatchOp.size(); i++) {
        if (miniBatchOp.getOperationStatus(i) == IGNORE) {
            continue;
        }
        Mutation m = miniBatchOp.getOperation(i);
        // skip this mutation if we aren't enabling indexing
        if (!this.builder.isEnabled(m)) {
            continue;
        }
        // Unless we're replaying edits to rebuild the index, we update the time stamp
        // of the data table to prevent overlapping time stamps (which prevents index
        // inconsistencies as this case isn't handled correctly currently).
        setTimestamp(m, now);
        if (m instanceof Put) {
            ImmutableBytesPtr rowKeyPtr = new ImmutableBytesPtr(m.getRow());
            Pair<Put, Put> dataRowState = context.dataRowStates.get(rowKeyPtr);
            if (dataRowState == null) {
                dataRowState = new Pair<Put, Put>(null, null);
                context.dataRowStates.put(rowKeyPtr, dataRowState);
            }
            Put nextDataRowState = dataRowState.getSecond();
            dataRowState.setSecond((nextDataRowState != null) ? applyNew((Put) m, nextDataRowState) : new Put((Put) m));
        }
    }
}
 

@Override
public List<Cell> getCurrentRowState(
        Mutation mutation,
        Collection<? extends ColumnReference> columnReferences,
        boolean ignoreNewerMutations) throws IOException {

    if(ignoreNewerMutations) {
        return doScan(mutation, columnReferences);
    }

    byte[] rowKey = mutation.getRow();
    return this.rowKeyPtrToCells.get(new ImmutableBytesPtr(rowKey));
}
 

private Mutation getDeleteMutation(Mutation orig, Long ts) {
    Mutation m = new Delete(orig.getRow());
    List<Cell> origList = orig.getFamilyCellMap().firstEntry().getValue();
    ts = ts == null ? EnvironmentEdgeManager.currentTimeMillis() : ts;
    Cell c = getNewPutCell(orig, origList, ts, KeyValue.Type.DeleteFamilyVersion);
    Cell empty = getEmptyCell(orig, origList, ts, KeyValue.Type.Put, true);
    byte[] fam = CellUtil.cloneFamily(origList.get(0));
    List<Cell> famCells = Lists.newArrayList();
    m.getFamilyCellMap().put(fam, famCells);
    famCells.add(c);
    famCells.add(empty);
    return m;
}
 

private void testIndexRowKeyBuilding(String schemaName, String tableName, String dataColumns,
        String pk, String indexColumns, Object[] values, String includeColumns,
        String dataProps, String indexProps, KeyValueBuilder builder) throws Exception {
    Connection conn = DriverManager.getConnection(getUrl());
    String fullTableName = SchemaUtil.getTableName(schemaName, tableName) ;
    conn.createStatement().execute("CREATE TABLE " + fullTableName + "(" + dataColumns + " CONSTRAINT pk PRIMARY KEY (" + pk + "))  " + (dataProps.isEmpty() ? "" : dataProps) );
    try {
        conn.createStatement().execute("CREATE INDEX idx ON " + fullTableName + "(" + indexColumns + ") " + (includeColumns.isEmpty() ? "" : "INCLUDE (" + includeColumns + ") ") + (indexProps.isEmpty() ? "" : indexProps));
        PTable table = conn.unwrap(PhoenixConnection.class).getPMetaData().getTable(SchemaUtil.getTableName(SchemaUtil.normalizeIdentifier(schemaName),SchemaUtil.normalizeIdentifier(tableName)));
        PTable index = conn.unwrap(PhoenixConnection.class).getPMetaData().getTable(SchemaUtil.getTableName(SchemaUtil.normalizeIdentifier(schemaName),SchemaUtil.normalizeIdentifier("idx")));
        ImmutableBytesWritable ptr = new ImmutableBytesWritable();
        table.getIndexMaintainers(ptr);
        List<IndexMaintainer> c1 = IndexMaintainer.deserialize(ptr, builder);
        assertEquals(1,c1.size());
        IndexMaintainer im1 = c1.get(0);
        
        StringBuilder buf = new StringBuilder("UPSERT INTO " + fullTableName  + " VALUES(");
        for (int i = 0; i < values.length; i++) {
            buf.append("?,");
        }
        buf.setCharAt(buf.length()-1, ')');
        PreparedStatement stmt = conn.prepareStatement(buf.toString());
        for (int i = 0; i < values.length; i++) {
            stmt.setObject(i+1, values[i]);
        }
        stmt.execute();
        	Iterator<Pair<byte[],List<KeyValue>>> iterator = PhoenixRuntime.getUncommittedDataIterator(conn);
        List<KeyValue> dataKeyValues = iterator.next().getSecond();
        Map<ColumnReference,byte[]> valueMap = Maps.newHashMapWithExpectedSize(dataKeyValues.size());
        ImmutableBytesWritable rowKeyPtr = new ImmutableBytesWritable(dataKeyValues.get(0).getRow());
        Put dataMutation = new Put(rowKeyPtr.copyBytes());
        for (KeyValue kv : dataKeyValues) {
            valueMap.put(new ColumnReference(kv.getFamily(),kv.getQualifier()), kv.getValue());
            dataMutation.add(kv);
        }
        ValueGetter valueGetter = newValueGetter(valueMap);
        
        List<Mutation> indexMutations =
                IndexTestUtil.generateIndexData(index, table, dataMutation, ptr, builder);
        assertEquals(1,indexMutations.size());
        assertTrue(indexMutations.get(0) instanceof Put);
        Mutation indexMutation = indexMutations.get(0);
        ImmutableBytesWritable indexKeyPtr = new ImmutableBytesWritable(indexMutation.getRow());
        
        ptr.set(rowKeyPtr.get(), rowKeyPtr.getOffset(), rowKeyPtr.getLength());
        byte[] mutablelndexRowKey = im1.buildRowKey(valueGetter, ptr);
        byte[] immutableIndexRowKey = indexKeyPtr.copyBytes();
        assertArrayEquals(immutableIndexRowKey, mutablelndexRowKey);
        
        for (ColumnReference ref : im1.getCoverededColumns()) {
            valueMap.get(ref);
        }
    } finally {
        try {
            conn.createStatement().execute("DROP TABLE " + fullTableName);
        } finally {
            conn.close();
        }
    }
}
 

/**
  * Add the necessary mutations for the pending batch on the local state. Handles rolling up
  * through history to determine the index changes after applying the batch (for the case where the
  * batch is back in time).
  * @param updateMap to update with index mutations
  * @param batch to apply to the current state
  * @param state current state of the table
  * @return the minimum timestamp across all index columns requested. If
  *         {@link ColumnTracker#isNewestTime(long)} returns <tt>true</tt> on the returned
  *         timestamp, we know that this <i>was not a back-in-time update</i>.
* @throws IOException 
  */
 private long
     addCurrentStateMutationsForBatch(IndexUpdateManager updateMap, LocalTableState state) throws IOException {

   // get the index updates for this current batch
   Iterable<IndexUpdate> upserts = codec.getIndexUpserts(state);
   state.resetTrackedColumns();

   /*
    * go through all the pending updates. If we are sure that all the entries are the latest
    * timestamp, we can just add the index updates and move on. However, if there are columns that
    * we skip past (based on the timestamp of the batch), we need to roll back up the history.
    * Regardless of whether or not they are the latest timestamp, the entries here are going to be
    * correct for the current batch timestamp, so we add them to the updates. The only thing we
    * really care about it if we need to roll up the history and fix it as we go.
    */
   // timestamp of the next update we need to track
   long minTs = ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP;
   List<IndexedColumnGroup> columnHints = new ArrayList<IndexedColumnGroup>();
   for (IndexUpdate update : upserts) {
     // this is the one bit where we check the timestamps
     final ColumnTracker tracker = update.getIndexedColumns();
     long trackerTs = tracker.getTS();
     // update the next min TS we need to track
     if (trackerTs < minTs) {
       minTs = tracker.getTS();
     }
     // track index hints for the next round. Hint if we need an update for that column for the
     // next timestamp. These columns clearly won't need to update as we go through time as they
     // already match the most recent possible thing.
     boolean needsCleanup = false;
     if (tracker.hasNewerTimestamps()) {
       columnHints.add(tracker);
       // this update also needs to be cleaned up at the next timestamp because it not the latest.
       needsCleanup = true;
     }


     // only make the put if the index update has been setup
     if (update.isValid()) {
       byte[] table = update.getTableName();
       Mutation mutation = update.getUpdate();
       updateMap.addIndexUpdate(table, mutation);

       // only make the cleanup if we made a put and need cleanup
       if (needsCleanup) {
         // there is a TS for the interested columns that is greater than the columns in the
         // put. Therefore, we need to issue a delete at the same timestamp
         Delete d = new Delete(mutation.getRow());
         d.setTimestamp(tracker.getTS());
         updateMap.addIndexUpdate(table, d);
       }
     }
   }
   return minTs;
 }
 

private void testIndexRowKeyBuilding(String schemaName, String tableName, String dataColumns,
         String pk, String indexColumns, Object[] values, String includeColumns,
         String dataProps, String indexProps, KeyValueBuilder builder) throws Exception {
     Connection conn = DriverManager.getConnection(getUrl());
     String fullTableName = SchemaUtil.getTableName(SchemaUtil.normalizeIdentifier(schemaName),SchemaUtil.normalizeIdentifier(tableName));
     String fullIndexName = SchemaUtil.getTableName(SchemaUtil.normalizeIdentifier(schemaName),SchemaUtil.normalizeIdentifier("idx"));
     conn.createStatement().execute("CREATE TABLE " + fullTableName + "(" + dataColumns + " CONSTRAINT pk PRIMARY KEY (" + pk + "))  " + (dataProps.isEmpty() ? "" : dataProps) );
     try {
         conn.createStatement().execute("CREATE INDEX idx ON " + fullTableName + "(" + indexColumns + ") " + (includeColumns.isEmpty() ? "" : "INCLUDE (" + includeColumns + ") ") + (indexProps.isEmpty() ? "" : indexProps));
         PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
         PTable table = pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName));
         PTable index = pconn.getTable(new PTableKey(pconn.getTenantId(),fullIndexName));
         ImmutableBytesWritable ptr = new ImmutableBytesWritable();
         table.getIndexMaintainers(ptr, pconn);
         List<IndexMaintainer> c1 = IndexMaintainer.deserialize(ptr, builder, true);
         assertEquals(1,c1.size());
         IndexMaintainer im1 = c1.get(0);
         
         StringBuilder buf = new StringBuilder("UPSERT INTO " + fullTableName  + " VALUES(");
         for (int i = 0; i < values.length; i++) {
             buf.append("?,");
         }
         buf.setCharAt(buf.length()-1, ')');
         PreparedStatement stmt = conn.prepareStatement(buf.toString());
         for (int i = 0; i < values.length; i++) {
             stmt.setObject(i+1, values[i]);
         }
         stmt.execute();
         	Iterator<Pair<byte[],List<Cell>>> iterator = PhoenixRuntime.getUncommittedDataIterator(conn);
         List<Cell> dataKeyValues = iterator.next().getSecond();
         Map<ColumnReference,byte[]> valueMap = Maps.newHashMapWithExpectedSize(dataKeyValues.size());
ImmutableBytesWritable rowKeyPtr = new ImmutableBytesWritable(dataKeyValues.get(0).getRowArray(), dataKeyValues.get(0).getRowOffset(), dataKeyValues.get(0).getRowLength());
         byte[] row = rowKeyPtr.copyBytes();
         Put dataMutation = new Put(row);
         for (Cell kv : dataKeyValues) {
             valueMap.put(new ColumnReference(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()), CellUtil.cloneValue(kv));
             dataMutation.add(kv);
         }
         ValueGetter valueGetter = newValueGetter(row, valueMap);
         
         List<Mutation> indexMutations = IndexTestUtil.generateIndexData(index, table, dataMutation, ptr, builder);
         assertEquals(1,indexMutations.size());
         assertTrue(indexMutations.get(0) instanceof Put);
         Mutation indexMutation = indexMutations.get(0);
         ImmutableBytesWritable indexKeyPtr = new ImmutableBytesWritable(indexMutation.getRow());
         ptr.set(rowKeyPtr.get(), rowKeyPtr.getOffset(), rowKeyPtr.getLength());
         byte[] mutablelndexRowKey = im1.buildRowKey(valueGetter, ptr, null, null, HConstants.LATEST_TIMESTAMP);
         byte[] immutableIndexRowKey = indexKeyPtr.copyBytes();
         assertArrayEquals(immutableIndexRowKey, mutablelndexRowKey);
         for (ColumnReference ref : im1.getCoveredColumns()) {
             valueMap.get(ref);
         }
         byte[] dataRowKey = im1.buildDataRowKey(indexKeyPtr, null);
         assertArrayEquals(dataRowKey, CellUtil.cloneRow(dataKeyValues.get(0)));
     } finally {
         try {
             conn.rollback();
             conn.createStatement().execute("DROP TABLE " + fullTableName);
         } finally {
             conn.close();
         }
     }
 }
 

public Collection<Pair<Mutation, byte[]>> getIndexUpdates(Table htable, Iterator<? extends Mutation> mutationIterator) throws IOException, SQLException {

        if (!mutationIterator.hasNext()) {
            return Collections.emptyList();
        }

        List<IndexMaintainer> indexMaintainers = indexMetaData.getIndexMaintainers();
        ResultScanner currentScanner = null;
        // Collect up all mutations in batch
        Map<ImmutableBytesPtr, MultiMutation> mutations =
                new HashMap<ImmutableBytesPtr, MultiMutation>();
        // Collect the set of mutable ColumnReferences so that we can first
        // run a scan to get the current state. We'll need this to delete
        // the existing index rows.
        int estimatedSize = indexMaintainers.size() * 10;
        Set<ColumnReference> mutableColumns = Sets.newHashSetWithExpectedSize(estimatedSize);
        for (IndexMaintainer indexMaintainer : indexMaintainers) {
            // For transactional tables, we use an index maintainer
            // to aid in rollback if there's a KeyValue column in the index. The alternative would be
            // to hold on to all uncommitted index row keys (even ones already sent to HBase) on the
            // client side.
            Set<ColumnReference> allColumns = indexMaintainer.getAllColumns();
            mutableColumns.addAll(allColumns);
        }

        Mutation m = mutationIterator.next();
        Map<String,byte[]> updateAttributes = m.getAttributesMap();
        byte[] txRollbackAttribute = updateAttributes.get(PhoenixTransactionContext.TX_ROLLBACK_ATTRIBUTE_KEY);
        boolean isRollback = txRollbackAttribute!=null;
        
        boolean isImmutable = indexMetaData.isImmutableRows();
        Map<ImmutableBytesPtr, MultiMutation> findPriorValueMutations;
        if (isImmutable && !isRollback) {
            findPriorValueMutations = new HashMap<ImmutableBytesPtr, MultiMutation>();
        } else {
            findPriorValueMutations = mutations;
        }
        
        while (true) {
            // add the mutation to the batch set
            ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow());
            // if we have no non PK columns, no need to find the prior values
            if ( mutations != findPriorValueMutations && indexMetaData.requiresPriorRowState(m) ) {
                addMutation(findPriorValueMutations, row, m);
            }
            addMutation(mutations, row, m);
            
            if (!mutationIterator.hasNext()) {
                break;
            }
            m = mutationIterator.next();
        }
        
        Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>(mutations.size() * 2 * indexMaintainers.size());
        // Track if we have row keys with Delete mutations (or Puts that are
        // Tephra's Delete marker). If there are none, we don't need to do the scan for
        // prior versions, if there are, we do. Since rollbacks always have delete mutations,
        // this logic will work there too.
        if (!findPriorValueMutations.isEmpty()) {
            List<KeyRange> keys = Lists.newArrayListWithExpectedSize(mutations.size());
            for (ImmutableBytesPtr ptr : findPriorValueMutations.keySet()) {
                keys.add(PVarbinary.INSTANCE.getKeyRange(ptr.copyBytesIfNecessary()));
            }
            Scan scan = new Scan();
            // Project all mutable columns
            for (ColumnReference ref : mutableColumns) {
                scan.addColumn(ref.getFamily(), ref.getQualifier());
            }
            /*
             * Indexes inherit the storage scheme of the data table which means all the indexes have the same
             * storage scheme and empty key value qualifier. Note that this assumption would be broken if we start
             * supporting new indexes over existing data tables to have a different storage scheme than the data
             * table.
             */
            byte[] emptyKeyValueQualifier = indexMaintainers.get(0).getEmptyKeyValueQualifier();
            
            // Project empty key value column
            scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), emptyKeyValueQualifier);
            ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, null, true, -1);
            scanRanges.initializeScan(scan);
            Table txTable = indexMetaData.getTransactionContext().getTransactionalTable(htable, true);
            // For rollback, we need to see all versions, including
            // the last committed version as there may be multiple
            // checkpointed versions.
            SkipScanFilter filter = scanRanges.getSkipScanFilter();
            if (isRollback) {
                filter = new SkipScanFilter(filter,true);
                indexMetaData.getTransactionContext().setVisibilityLevel(PhoenixVisibilityLevel.SNAPSHOT_ALL);
            }
            scan.setFilter(filter);
            currentScanner = txTable.getScanner(scan);
        }
        if (isRollback) {
            processRollback(indexMetaData, txRollbackAttribute, currentScanner, mutableColumns, indexUpdates, mutations);
        } else {
            processMutation(indexMetaData, txRollbackAttribute, currentScanner, mutableColumns, indexUpdates, mutations, findPriorValueMutations);
        }
        
        return indexUpdates;
    }
 

private void filterIndexCheckerMutations(Map<TableInfo, List<Mutation>> mutationMap,
        Map<TableInfo, List<Mutation>> unverifiedIndexMutations,
        Map<TableInfo, List<Mutation>> verifiedOrDeletedIndexMutations) throws SQLException {
    Iterator<Entry<TableInfo, List<Mutation>>> mapIter = mutationMap.entrySet().iterator();

    while (mapIter.hasNext()) {
        Entry<TableInfo, List<Mutation>> pair = mapIter.next();
        TableInfo tableInfo = pair.getKey();
        if (!tableInfo.getPTable().getType().equals(PTableType.INDEX)) {
            continue;
        }
        PTable logicalTable = tableInfo.getPTable();
        if (tableInfo.getOrigTableRef().getTable().isImmutableRows()
                && IndexUtil.isGlobalIndexCheckerEnabled(connection,
                tableInfo.getHTableName())) {

            byte[] emptyCF = SchemaUtil.getEmptyColumnFamily(logicalTable);
            byte[] emptyCQ = EncodedColumnsUtil.getEmptyKeyValueInfo(logicalTable).getFirst();
            List<Mutation> mutations = pair.getValue();

            for (Mutation m : mutations) {
                if (m == null) {
                    continue;
                }
                if (m instanceof Delete) {
                    Put put = new Put(m.getRow());
                    put.addColumn(emptyCF, emptyCQ, IndexRegionObserver.getMaxTimestamp(m),
                            IndexRegionObserver.UNVERIFIED_BYTES);
                    // The Delete gets marked as unverified in Phase 1 and gets deleted on Phase 3.
                    addToMap(unverifiedIndexMutations, tableInfo, put);
                    addToMap(verifiedOrDeletedIndexMutations, tableInfo, m);
                } else if (m instanceof Put) {
                    long timestamp = IndexRegionObserver.getMaxTimestamp(m);

                    // Phase 1 index mutations are set to unverified
                    // Send entire mutation with the unverified status
                    // Remove the empty column prepared by Index codec as we need to change its value
                    IndexRegionObserver.removeEmptyColumn(m, emptyCF, emptyCQ);
                    ((Put) m).addColumn(emptyCF, emptyCQ, timestamp, IndexRegionObserver.UNVERIFIED_BYTES);
                    addToMap(unverifiedIndexMutations, tableInfo, m);

                    // Phase 3 mutations are verified
                    Put verifiedPut = new Put(m.getRow());
                    verifiedPut.addColumn(emptyCF, emptyCQ, timestamp, IndexRegionObserver.VERIFIED_BYTES);
                    addToMap(verifiedOrDeletedIndexMutations, tableInfo, verifiedPut);
                } else {
                    addToMap(unverifiedIndexMutations, tableInfo, m);
                }
            }

            mapIter.remove();
        }

    }
}
 

private void preparePostIndexMutations(BatchMutateContext context, long now, PhoenixIndexMetaData indexMetaData,
                                       String tableName)
        throws Throwable {
    context.postIndexUpdates = ArrayListMultimap.<HTableInterfaceReference, Mutation>create();
    List<IndexMaintainer> maintainers = indexMetaData.getIndexMaintainers();
    // Check if we need to skip post index update for any of the rows
    for (IndexMaintainer indexMaintainer : maintainers) {
        byte[] emptyCF = indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary();
        byte[] emptyCQ = indexMaintainer.getEmptyKeyValueQualifier();
        HTableInterfaceReference hTableInterfaceReference =
                new HTableInterfaceReference(new ImmutableBytesPtr(indexMaintainer.getIndexTableName()));
        List<Pair<Mutation, byte[]>> updates = context.indexUpdates.get(hTableInterfaceReference);
        for (Pair<Mutation, byte[]> update : updates) {
            // Are there concurrent updates on the data table row? if so, skip post index updates
            // and let read repair resolve conflicts
            ImmutableBytesPtr rowKey = new ImmutableBytesPtr(update.getSecond());
            PendingRow pendingRow = pendingRows.get(rowKey);
            if (!pendingRow.isConcurrent()) {
                Mutation m = update.getFirst();
                if (m instanceof Put) {
                    Put verifiedPut = new Put(m.getRow());
                    // Set the status of the index row to "verified"
                    verifiedPut.addColumn(emptyCF, emptyCQ, now, VERIFIED_BYTES);
                    context.postIndexUpdates.put(hTableInterfaceReference, verifiedPut);
                } else {
                    context.postIndexUpdates.put(hTableInterfaceReference, m);
                }
            } else {
                if (!hasAllIndexedColumns(indexMaintainer, context.multiMutationMap.get(rowKey))) {
                    // This batch needs to be retried since one of the concurrent mutations does not have the value
                    // for an indexed column. Not including an index column may lead to incorrect index row key
                    // generation for concurrent mutations since concurrent mutations are not serialized entirely
                    // and do not see each other's effect on data table. Throwing an IOException will result in
                    // retries of this batch. Before throwing exception, we need to remove reference counts and
                    // locks for the rows of this batch
                    removePendingRows(context);
                    context.indexUpdates.clear();
                    for (RowLock rowLock : context.rowLocks) {
                        rowLock.release();
                    }
                    context.rowLocks.clear();
                    throw new IOException("One of the concurrent mutations does not have all indexed columns. " +
                            "The batch needs to be retried " + tableName);
                }
            }
        }
    }

    // We are done with handling concurrent mutations. So we can remove the rows of this batch from
    // the collection of pending rows
    removePendingRows(context);
    context.indexUpdates.clear();
}
 

/**
 * This method applies pending delete mutations on the next row states
 */
private void applyPendingDeleteMutations(MiniBatchOperationInProgress<Mutation> miniBatchOp,
                                         BatchMutateContext context) throws IOException {
    for (int i = 0; i < miniBatchOp.size(); i++) {
        if (miniBatchOp.getOperationStatus(i) == IGNORE) {
            continue;
        }
        Mutation m = miniBatchOp.getOperation(i);
        if (!this.builder.isEnabled(m)) {
            continue;
        }
        if (!(m instanceof Delete)) {
            continue;
        }
        ImmutableBytesPtr rowKeyPtr = new ImmutableBytesPtr(m.getRow());
        Pair<Put, Put> dataRowState = context.dataRowStates.get(rowKeyPtr);
        if (dataRowState == null) {
            dataRowState = new Pair<Put, Put>(null, null);
            context.dataRowStates.put(rowKeyPtr, dataRowState);
        }
        Put nextDataRowState = dataRowState.getSecond();
        if (nextDataRowState == null) {
            if (dataRowState.getFirst() == null) {
                // This is a delete row mutation on a non-existing row. There is no need to apply this mutation
                // on the data table
                miniBatchOp.setOperationStatus(i, NOWRITE);
            }
            continue;
        }
        for (List<Cell> cells : m.getFamilyCellMap().values()) {
            for (Cell cell : cells) {
                switch (KeyValue.Type.codeToType(cell.getTypeByte())) {
                    case DeleteFamily:
                    case DeleteFamilyVersion:
                        nextDataRowState.getFamilyCellMap().remove(CellUtil.cloneFamily(cell));
                        break;
                    case DeleteColumn:
                    case Delete:
                        removeColumn(nextDataRowState, cell);
                }
            }
        }
        if (nextDataRowState != null && nextDataRowState.getFamilyCellMap().size() == 0) {
            dataRowState.setSecond(null);
        }
    }
}
 

@SuppressWarnings("deprecation")
 private void testIndexRowKeyBuilding(String schemaName, String tableName, String dataColumns,
         String pk, String indexColumns, Object[] values, String includeColumns,
         String dataProps, String indexProps, KeyValueBuilder builder) throws Exception {
     Connection conn = DriverManager.getConnection(getUrl());
     String fullTableName = SchemaUtil.getTableName(SchemaUtil.normalizeIdentifier(schemaName),SchemaUtil.normalizeIdentifier(tableName));
     String fullIndexName = SchemaUtil.getTableName(SchemaUtil.normalizeIdentifier(schemaName),SchemaUtil.normalizeIdentifier("idx"));
     conn.createStatement().execute("CREATE TABLE " + fullTableName + "(" + dataColumns + " CONSTRAINT pk PRIMARY KEY (" + pk + "))  " + (dataProps.isEmpty() ? "" : dataProps) );
     try {
         conn.createStatement().execute("CREATE INDEX idx ON " + fullTableName + "(" + indexColumns + ") " + (includeColumns.isEmpty() ? "" : "INCLUDE (" + includeColumns + ") ") + (indexProps.isEmpty() ? "" : indexProps));
         PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
         PTable table = pconn.getMetaDataCache().getTable(new PTableKey(pconn.getTenantId(), fullTableName));
         PTable index = pconn.getMetaDataCache().getTable(new PTableKey(pconn.getTenantId(),fullIndexName));
         ImmutableBytesWritable ptr = new ImmutableBytesWritable();
         table.getIndexMaintainers(ptr, pconn);
         List<IndexMaintainer> c1 = IndexMaintainer.deserialize(ptr, builder);
         assertEquals(1,c1.size());
         IndexMaintainer im1 = c1.get(0);
         
         StringBuilder buf = new StringBuilder("UPSERT INTO " + fullTableName  + " VALUES(");
         for (int i = 0; i < values.length; i++) {
             buf.append("?,");
         }
         buf.setCharAt(buf.length()-1, ')');
         PreparedStatement stmt = conn.prepareStatement(buf.toString());
         for (int i = 0; i < values.length; i++) {
             stmt.setObject(i+1, values[i]);
         }
         stmt.execute();
         	Iterator<Pair<byte[],List<KeyValue>>> iterator = PhoenixRuntime.getUncommittedDataIterator(conn);
         List<KeyValue> dataKeyValues = iterator.next().getSecond();
         Map<ColumnReference,byte[]> valueMap = Maps.newHashMapWithExpectedSize(dataKeyValues.size());
         byte[] row = dataKeyValues.get(0).getRow();
ImmutableBytesWritable rowKeyPtr = new ImmutableBytesWritable(row);
         Put dataMutation = new Put(rowKeyPtr.copyBytes());
         for (KeyValue kv : dataKeyValues) {
             valueMap.put(new ColumnReference(kv.getFamily(),kv.getQualifier()), kv.getValue());
             dataMutation.add(kv);
         }
         ValueGetter valueGetter = newValueGetter(row, valueMap);
         
         List<Mutation> indexMutations =
                 IndexTestUtil.generateIndexData(index, table, dataMutation, ptr, builder);
         assertEquals(1,indexMutations.size());
         assertTrue(indexMutations.get(0) instanceof Put);
         Mutation indexMutation = indexMutations.get(0);
         ImmutableBytesWritable indexKeyPtr = new ImmutableBytesWritable(indexMutation.getRow());
         ptr.set(rowKeyPtr.get(), rowKeyPtr.getOffset(), rowKeyPtr.getLength());
         byte[] mutablelndexRowKey = im1.buildRowKey(valueGetter, ptr, null, null);
         byte[] immutableIndexRowKey = indexKeyPtr.copyBytes();
         assertArrayEquals(immutableIndexRowKey, mutablelndexRowKey);
         for (ColumnReference ref : im1.getCoverededColumns()) {
             valueMap.get(ref);
         }
         byte[] dataRowKey = im1.buildDataRowKey(indexKeyPtr, null);
         assertArrayEquals(dataRowKey, dataKeyValues.get(0).getRow());
     } finally {
         try {
             conn.createStatement().execute("DROP TABLE " + fullTableName);
         } finally {
             conn.close();
         }
     }
 }
 

/**
  * Add the necessary mutations for the pending batch on the local state. Handles rolling up
  * through history to determine the index changes after applying the batch (for the case where the
  * batch is back in time).
  * @param updateMap to update with index mutations
  * @param batch to apply to the current state
  * @param state current state of the table
  * @return the minimum timestamp across all index columns requested. If
  *         {@link ColumnTracker#isNewestTime(long)} returns <tt>true</tt> on the returned
  *         timestamp, we know that this <i>was not a back-in-time update</i>.
* @throws IOException 
  */
 private long
     addCurrentStateMutationsForBatch(IndexUpdateManager updateMap, LocalTableState state) throws IOException {

   // get the index updates for this current batch
   Iterable<IndexUpdate> upserts = codec.getIndexUpserts(state);
   state.resetTrackedColumns();

   /*
    * go through all the pending updates. If we are sure that all the entries are the latest
    * timestamp, we can just add the index updates and move on. However, if there are columns that
    * we skip past (based on the timestamp of the batch), we need to roll back up the history.
    * Regardless of whether or not they are the latest timestamp, the entries here are going to be
    * correct for the current batch timestamp, so we add them to the updates. The only thing we
    * really care about it if we need to roll up the history and fix it as we go.
    */
   // timestamp of the next update we need to track
   long minTs = ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP;
   List<IndexedColumnGroup> columnHints = new ArrayList<IndexedColumnGroup>();
   for (IndexUpdate update : upserts) {
     // this is the one bit where we check the timestamps
     final ColumnTracker tracker = update.getIndexedColumns();
     long trackerTs = tracker.getTS();
     // update the next min TS we need to track
     if (trackerTs < minTs) {
       minTs = tracker.getTS();
     }
     // track index hints for the next round. Hint if we need an update for that column for the
     // next timestamp. These columns clearly won't need to update as we go through time as they
     // already match the most recent possible thing.
     boolean needsCleanup = false;
     if (tracker.hasNewerTimestamps()) {
       columnHints.add(tracker);
       // this update also needs to be cleaned up at the next timestamp because it not the latest.
       needsCleanup = true;
     }


     // only make the put if the index update has been setup
     if (update.isValid()) {
       byte[] table = update.getTableName();
       Mutation mutation = update.getUpdate();
       updateMap.addIndexUpdate(table, mutation);

       // only make the cleanup if we made a put and need cleanup
       if (needsCleanup) {
         // there is a TS for the interested columns that is greater than the columns in the
         // put. Therefore, we need to issue a delete at the same timestamp
         Delete d = new Delete(mutation.getRow());
         d.setTimestamp(tracker.getTS());
         updateMap.addIndexUpdate(table, d);
       }
     }
   }
   return minTs;
 }