Java Code Examples for org.apache.hadoop.hbase.CellUtil#isDelete()

private int countDeleteMarkers(HRegion region) throws IOException {
  Scan s = new Scan();
  s.setRaw(true);
  // use max versions from the store(s)
  s.readVersions(region.getStores().iterator().next().getScanInfo().getMaxVersions());
  InternalScanner scan = region.getScanner(s);
  List<Cell> kvs = new ArrayList<>();
  int res = 0;
  boolean hasMore;
  do {
    hasMore = scan.next(kvs);
    for (Cell kv : kvs) {
      if(CellUtil.isDelete(kv)) {
        res++;
      }
    }
    kvs.clear();
  } while (hasMore);
  scan.close();
  return res;
}
 

/**
 * Makes a HBase Result object based on the KeyValue's from the SEP event. Usually, this will only be used in
 * situations where only new data is written (or updates are complete row updates), so we don't expect any
 * delete-type key-values, but just to be sure we filter them out.
 */
@Override
public Result toResult() {

    List<Cell> filteredKeyValues = Lists.newArrayListWithCapacity(sepEvent.getKeyValues().size());

    for (Cell kv : getKeyValues()) {
        if (!CellUtil.isDelete(kv) && !CellUtil.isDeleteFamily(kv)) {
            filteredKeyValues.add(kv);
        }
    }

    // A Result object requires that the KeyValues are sorted (e.g., it does binary search on them)
    Collections.sort(filteredKeyValues, KeyValue.COMPARATOR);
    return Result.create(filteredKeyValues);
}
 

/**
 * Calculate a map of Solr document ids to relevant RowData, only taking the most recent event for each document id..
 */
private Map<String, RowData> calculateUniqueEvents(List<RowData> rowDataList) {
    Map<String, RowData> idToEvent = Maps.newHashMap();
    for (RowData rowData : rowDataList) {
        // Check if the event contains changes to relevant key values
        boolean relevant = false;
        for (Cell kv : rowData.getKeyValues()) {
            if (mapper.isRelevantKV((KeyValue)kv) || CellUtil.isDelete(kv)) {
                relevant = true;
                break;
            }
        }

        if (!relevant) {
            continue;
        }
        if (uniqueKeyFormatter instanceof UniqueTableKeyFormatter) {
            idToEvent.put(((UniqueTableKeyFormatter) uniqueKeyFormatter).formatRow(rowData.getRow(),
                    rowData.getTable()), rowData);
        } else {
            idToEvent.put(uniqueKeyFormatter.formatRow(rowData.getRow()), rowData);
        }

    }
    return idToEvent;
}
 

private boolean isSatisfiedMidKeyCondition(Cell kv) {
    if (CellUtil.isDelete(kv) && kv.getValueLength() == 0) {
        // In case of a Delete type KV, let it be going to both the daughter regions.
        // No problems in doing so. In the correct daughter region where it belongs to, this delete
        // tomb will really delete a KV. In the other it will just hang around there with no actual
        // kv coming for which this is a delete tomb. :)
        return true;
    }
    ImmutableBytesWritable rowKey =
            new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset() + offset,
                    kv.getRowLength() - offset);
    Entry<ImmutableBytesWritable, IndexMaintainer> entry = indexMaintainers.entrySet().iterator().next();
    IndexMaintainer indexMaintainer = entry.getValue();
    byte[] viewIndexId = indexMaintainer.getViewIndexIdFromIndexRowKey(rowKey);
    IndexMaintainer actualIndexMaintainer = indexMaintainers.get(new ImmutableBytesWritable(viewIndexId));
    byte[] dataRowKey = actualIndexMaintainer.buildDataRowKey(rowKey, this.viewConstants);
    int compareResult = Bytes.compareTo(dataRowKey, splitRow);
    if (top) {
        if (compareResult >= 0) {
            return true;
        }
    } else {
        if (compareResult < 0) {
            return true;
        }
    }
    return false;
}
 

public static MutationProto toMutation(final MutationType type, final Mutation mutation,
    MutationProto.Builder builder, long nonce)
throws IOException {
  builder = getMutationBuilderAndSetCommonFields(type, mutation, builder);
  if (nonce != HConstants.NO_NONCE) {
    builder.setNonce(nonce);
  }
  if (type == MutationType.INCREMENT) {
    builder.setTimeRange(ProtobufUtil.toTimeRange(((Increment) mutation).getTimeRange()));
  }
  if (type == MutationType.APPEND) {
    builder.setTimeRange(ProtobufUtil.toTimeRange(((Append) mutation).getTimeRange()));
  }
  ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
  QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
  for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
    columnBuilder.clear();
    columnBuilder.setFamily(UnsafeByteOperations.unsafeWrap(family.getKey()));
    for (Cell cell: family.getValue()) {
      valueBuilder.clear();
      valueBuilder.setQualifier(UnsafeByteOperations.unsafeWrap(
          cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
      valueBuilder.setValue(UnsafeByteOperations.unsafeWrap(
          cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
      valueBuilder.setTimestamp(cell.getTimestamp());
      if (type == MutationType.DELETE || (type == MutationType.PUT && CellUtil.isDelete(cell))) {
        KeyValue.Type keyValueType = KeyValue.Type.codeToType(cell.getTypeByte());
        valueBuilder.setDeleteType(toDeleteType(keyValueType));
      }
      columnBuilder.addQualifierValue(valueBuilder.build());
    }
    builder.addColumnValue(columnBuilder.build());
  }
  return builder.build();
}
 

@Override
protected void calculateIndexUpdates(List<RowData> rowDataList, SolrUpdateCollector updateCollector) throws IOException {
    Map<String, KeyValue> idToKeyValue = calculateUniqueEvents(rowDataList);
    for (Entry<String, KeyValue> idToKvEntry : idToKeyValue.entrySet()) {
        String documentId = idToKvEntry.getKey();

        KeyValue keyValue = idToKvEntry.getValue();
        if (CellUtil.isDelete(keyValue)) {
            handleDelete(documentId, keyValue, updateCollector, uniqueKeyFormatter);
        } else {
            Result result = Result.create(Collections.<Cell>singletonList(keyValue));
            SolrUpdateWriter updateWriter = new RowAndFamilyAddingSolrUpdateWriter(
                    conf.getRowField(),
                    conf.getColumnFamilyField(),
                    uniqueKeyFormatter,
                    keyValue,
                    new IdAddingSolrUpdateWriter(
                            conf.getUniqueKeyField(),
                            documentId,
                            conf.getTableNameField(),
                            tableName,
                            updateCollector));

            mapper.map(result, updateWriter);

        }
    }
}
 

@VisibleForTesting
public boolean shouldRetainNonTransactionallyDeletedCell(Cell cell) {
    return (CellUtil.isDelete(cell) || CellUtil.isDeleteFamily(cell))
            &&
            retainNonTransactionallyDeletedCells;
}
 

@Test(timeOut = 60_000)
public void testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs() throws Throwable {
    String TEST_TABLE = "testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs";
    createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
    TTable txTable = new TTable(connection, TEST_TABLE);

    Table table = txTable.getHTable();

    // Configure the environment to create a minor compaction

    // Write first a value transactionally
    HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
    byte[] rowId = Bytes.toBytes("row1");
    Put p0 = new Put(rowId);
    p0.addColumn(fam, qual, Bytes.toBytes("testValue-0"));
    txTable.put(tx0, p0);
    tm.commit(tx0);

    // create the first hfile
    manualFlush(TEST_TABLE);

    // Write another value transactionally
    HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
    Put p1 = new Put(rowId);
    p1.addColumn(fam, qual, Bytes.toBytes("testValue-1"));
    txTable.put(tx1, p1);
    tm.commit(tx1);

    // create the second hfile
    manualFlush(TEST_TABLE);

    // Write yet another value transactionally
    HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
    Put p2 = new Put(rowId);
    p2.addColumn(fam, qual, Bytes.toBytes("testValue-2"));
    txTable.put(tx2, p2);
    tm.commit(tx2);

    // create a third hfile
    manualFlush(TEST_TABLE);

    // Then perform a non-transactional Delete
    Delete d = new Delete(rowId);
    d.addColumn(fam, qual);
    table.delete(d);

    // create the fourth hfile
    manualFlush(TEST_TABLE);

    // Trigger the minor compaction
    HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
    setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
    admin.compact(TableName.valueOf(TEST_TABLE));
    Thread.sleep(5000);

    // Then perform a non-tx (raw) scan...
    Scan scan = new Scan();
    scan.setRaw(true);
    ResultScanner scannerResults = table.getScanner(scan);

    // ...and test the deleted cell is still there
    int count = 0;
    Result scanResult;
    List<Cell> listOfCellsScanned = new ArrayList<>();
    while ((scanResult = scannerResults.next()) != null) {
        listOfCellsScanned = scanResult.listCells(); // equivalent to rawCells()
        count++;
    }
    assertEquals(count, 1, "There should be only one result in scan results");
    assertEquals(listOfCellsScanned.size(), 3, "There should be 3 cell entries in scan results (2 puts, 1 del)");
    boolean wasDeletedCellFound = false;
    int numberOfDeletedCellsFound = 0;
    for (Cell cell : listOfCellsScanned) {
        if (CellUtil.isDelete(cell)) {
            wasDeletedCellFound = true;
            numberOfDeletedCellsFound++;
        }
    }
    assertTrue(wasDeletedCellFound, "We should have found a non-transactionally deleted cell");
    assertEquals(numberOfDeletedCellsFound, 1, "There should be only only one deleted cell");

    table.close();
}