Java Code Examples for org.apache.hadoop.hbase.regionserver.HRegion#flushcache()

The following examples show how to use org.apache.hadoop.hbase.regionserver.HRegion#flushcache() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDeleteFiltering() throws Exception {
  String tableName = "TestDeleteFiltering";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, 0);
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    byte[] row = Bytes.toBytes(1);
    for (int i = 4; i < V.length; i++) {
      Put p = new Put(row);
      p.addColumn(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
      region.put(p);
    }

    // delete from the third entry back
    // take that cell's timestamp + 1 to simulate a delete in a new tx
    long deleteTs = V[5] + 1;
    Delete d = new Delete(row, deleteTs);
    LOG.info("Issuing delete at timestamp " + deleteTs);
    // row deletes are not yet supported (TransactionAwareHTable normally handles this)
    d.addColumns(familyBytes, columnBytes);
    region.delete(d);

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, we should drop the deleted version, but not the others
    LOG.info("Flushing region " + region.getRegionInfo().getRegionNameAsString());
    region.flushcache(true, false, new FlushLifeCycleTracker() { });

    // now a normal scan should return row with versions at: V[8], V[6].
    // V[7] is invalid and V[5] and prior are deleted.
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);
    // should be only one row
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 1,
        new long[]{V[8], V[6], deleteTs},
        new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
  } finally {
    region.close();
  }
}

Example 2

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDeleteFiltering() throws Exception {
  String tableName = "TestDeleteFiltering";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, 0);
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    byte[] row = Bytes.toBytes(1);
    for (int i = 4; i < V.length; i++) {
      Put p = new Put(row);
      p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
      region.put(p);
    }

    // delete from the third entry back
    // take that cell's timestamp + 1 to simulate a delete in a new tx
    long deleteTs = V[5] + 1;
    Delete d = new Delete(row, deleteTs);
    LOG.info("Issuing delete at timestamp " + deleteTs);
    // row deletes are not yet supported (TransactionAwareHTable normally handles this)
    d.deleteColumns(familyBytes, columnBytes);
    region.delete(d);

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, we should drop the deleted version, but not the others
    LOG.info("Flushing region " + region.getRegionInfo().getRegionNameAsString());
    region.flushcache(true, false);

    // now a normal scan should return row with versions at: V[8], V[6].
    // V[7] is invalid and V[5] and prior are deleted.
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);
    // should be only one row
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 1,
        new long[]{V[8], V[6], deleteTs},
        new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
  } finally {
    region.close();
  }
}

Example 3

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDataJanitorRegionScanner() throws Exception {
  String tableName = "TestRegionScanner";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    for (int i = 1; i <= 8; i++) {
      for (int k = 1; k <= i; k++) {
        Put p = new Put(Bytes.toBytes(i));
        p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
        region.put(p);
      }
    }

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
    LOG.info("Flushing region " + region.getRegionInfo().getRegionNameAsString());
    Region.FlushResult flushResult = region.flushcache(true, false);
    Assert.assertTrue("Unexpected flush result: " + flushResult, flushResult.isFlushSucceeded());
    
    // now a normal scan should only return the valid rows
    // do not use a filter here to test that cleanup works on flush
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);

    // first returned value should be "4" with version "4"
    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 4, new long[]{V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 5, new long[] {V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 6, new long[]{V[6], V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 7, new long[]{V[6], V[4]});

    results.clear();
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
  } finally {
    region.close();
  }
}

Example 4

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testPreExistingData() throws Exception {
  String tableName = "TestPreExistingData";
  byte[] familyBytes = Bytes.toBytes("f");
  long ttlMillis = TimeUnit.DAYS.toMillis(14);
  HRegion region = createRegion(tableName, familyBytes, ttlMillis);
  try {
    region.initialize();

    // timestamps for pre-existing, non-transactional data
    long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
    long older = now - ttlMillis / 2;
    long newer = now - ttlMillis / 3;
    // timestamps for transactional data
    long nowTx = txVisibilityState.getVisibilityUpperBound();
    long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
    long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;

    Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    ttls.put(familyBytes, ttlMillis);

    List<Cell> cells = new ArrayList<>();
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));

    // Write non-transactional and transactional data
    for (Cell c : cells) {
      region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
    }

    Scan rawScan = new Scan();
    rawScan.setMaxVersions();

    Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
    Scan txScan = new Scan();
    txScan.setMaxVersions();
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // read all back with raw scanner
    scanAndAssert(region, cells, rawScan);

    // read all back with transaction filter
    scanAndAssert(region, cells, txScan);

    // force a flush to clear the memstore
    region.flushcache();
    scanAndAssert(region, cells, txScan);

    // force a major compaction to remove any expired cells
    region.compactStores(true);
    scanAndAssert(region, cells, txScan);

    // Reduce TTL, this should make cells with timestamps older and olderTx expire
    long newTtl = ttlMillis / 2 - 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // Raw scan should still give all cells
    scanAndAssert(region, cells, rawScan);
    // However, tx scan should not return expired cells
    scanAndAssert(region, select(cells, 1, 3, 5), txScan);

    region.flushcache();
    scanAndAssert(region, cells, rawScan);

    // force a major compaction to remove any expired cells
    region.compactStores(true);
    // This time raw scan too should not return expired cells, as they would be dropped during major compaction
    scanAndAssert(region, select(cells, 1, 3, 5), rawScan);

    // Reduce TTL again to 1 ms, this should expire all cells
    newTtl = 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // force a major compaction to remove expired cells
    region.compactStores(true);
    // This time raw scan should not return any cells, as all cells have expired.
    scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
  } finally {
    region.close();
  }
}

Example 5

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDataJanitorRegionScanner() throws Exception {
  String tableName = "TestRegionScanner";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    for (int i = 1; i <= 8; i++) {
      for (int k = 1; k <= i; k++) {
        Put p = new Put(Bytes.toBytes(i));
        p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
        region.put(p);
      }
    }

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
    LOG.info("Flushing region " + region.getRegionNameAsString());
    HRegion.FlushResult flushResult = region.flushcache();
    Assert.assertTrue("Unexpected flush result: " + flushResult, flushResult.isFlushSucceeded());

    // now a normal scan should only return the valid rows
    // do not use a filter here to test that cleanup works on flush
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);

    // first returned value should be "4" with version "4"
    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 4, new long[] {V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 5, new long[] {V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 6, new long[] {V[6], V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 7, new long[] {V[6], V[4]});

    results.clear();
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
  } finally {
    region.close();
  }
}

Example 6

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDataJanitorRegionScanner() throws Exception {
  String tableName = "TestRegionScanner";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    for (int i = 1; i <= 8; i++) {
      for (int k = 1; k <= i; k++) {
        Put p = new Put(Bytes.toBytes(i));
        p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
        region.put(p);
      }
    }

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
    LOG.info("Flushing region " + region.getRegionNameAsString());
    HRegion.FlushResult flushResult = region.flushcache();
    Assert.assertTrue("Unexpected flush result: " + flushResult, flushResult.isFlushSucceeded());

    // now a normal scan should only return the valid rows
    // do not use a filter here to test that cleanup works on flush
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);

    // first returned value should be "4" with version "4"
    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 4, new long[]{V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 5, new long[] {V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 6, new long[]{V[6], V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 7, new long[]{V[6], V[4]});

    results.clear();
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
  } finally {
    region.close();
  }
}

Example 7

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testPreExistingData() throws Exception {
  String tableName = "TestPreExistingData";
  byte[] familyBytes = Bytes.toBytes("f");
  long ttlMillis = TimeUnit.DAYS.toMillis(14);
  HRegion region = createRegion(tableName, familyBytes, ttlMillis);
  try {
    region.initialize();

    // timestamps for pre-existing, non-transactional data
    long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
    long older = now - ttlMillis / 2;
    long newer = now - ttlMillis / 3;
    // timestamps for transactional data
    long nowTx = txVisibilityState.getVisibilityUpperBound();
    long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
    long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;

    Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    ttls.put(familyBytes, ttlMillis);

    List<Cell> cells = new ArrayList<>();
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));

    // Write non-transactional and transactional data
    for (Cell c : cells) {
      region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
    }

    Scan rawScan = new Scan();
    rawScan.setMaxVersions();

    Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
    Scan txScan = new Scan();
    txScan.setMaxVersions();
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // read all back with raw scanner
    scanAndAssert(region, cells, rawScan);

    // read all back with transaction filter
    scanAndAssert(region, cells, txScan);

    // force a flush to clear the memstore
    region.flushcache();
    scanAndAssert(region, cells, txScan);

    // force a major compaction to remove any expired cells
    region.compactStores(true);
    scanAndAssert(region, cells, txScan);

    // Reduce TTL, this should make cells with timestamps older and olderTx expire
    long newTtl = ttlMillis / 2 - 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // Raw scan should still give all cells
    scanAndAssert(region, cells, rawScan);
    // However, tx scan should not return expired cells
    scanAndAssert(region, select(cells, 1, 3, 5), txScan);

    region.flushcache();
    scanAndAssert(region, cells, rawScan);

    // force a major compaction to remove any expired cells
    region.compactStores(true);
    // This time raw scan too should not return expired cells, as they would be dropped during major compaction
    scanAndAssert(region, select(cells, 1, 3, 5), rawScan);

    // Reduce TTL again to 1 ms, this should expire all cells
    newTtl = 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // force a major compaction to remove expired cells
    region.compactStores(true);
    // This time raw scan should not return any cells, as all cells have expired.
    scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
  } finally {
    region.close();
  }
}

Example 8

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDeleteFiltering() throws Exception {
  String tableName = "TestDeleteFiltering";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, 0);
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    byte[] row = Bytes.toBytes(1);
    for (int i = 4; i < V.length; i++) {
      Put p = new Put(row);
      p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
      region.put(p);
    }

    // delete from the third entry back
    // take that cell's timestamp + 1 to simulate a delete in a new tx
    long deleteTs = V[5] + 1;
    Delete d = new Delete(row, deleteTs);
    LOG.info("Issuing delete at timestamp " + deleteTs);
    // row deletes are not yet supported (TransactionAwareHTable normally handles this)
    d.deleteColumns(familyBytes, columnBytes);
    region.delete(d);

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, we should drop the deleted version, but not the others
    LOG.info("Flushing region " + region.getRegionNameAsString());
    region.flushcache();

    // now a normal scan should return row with versions at: V[8], V[6].
    // V[7] is invalid and V[5] and prior are deleted.
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);
    // should be only one row
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 1,
                          new long[]{V[8], V[6], deleteTs},
                          new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
  } finally {
    region.close();
  }
}

Example 9

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDataJanitorRegionScanner() throws Exception {
  String tableName = "TestRegionScanner";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    for (int i = 1; i <= 8; i++) {
      for (int k = 1; k <= i; k++) {
        Put p = new Put(Bytes.toBytes(i));
        p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
        region.put(p);
      }
    }

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
    LOG.info("Flushing region " + region.getRegionNameAsString());
    HRegion.FlushResult flushResult = region.flushcache();
    Assert.assertTrue("Unexpected flush result: " + flushResult.toString(), flushResult.isFlushSucceeded());

    // now a normal scan should only return the valid rows
    // do not use a filter here to test that cleanup works on flush
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);

    // first returned value should be "4" with version "4"
    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 4, new long[] {V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 5, new long[] {V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 6, new long[] {V[6], V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 7, new long[] {V[6], V[4]});

    results.clear();
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
  } finally {
    region.close();
  }
}

Example 10

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testPreExistingData() throws Exception {
  String tableName = "TestPreExistingData";
  byte[] familyBytes = Bytes.toBytes("f");
  long ttlMillis = TimeUnit.DAYS.toMillis(14);
  HRegion region = createRegion(tableName, familyBytes, ttlMillis);
  try {
    region.initialize();

    // timestamps for pre-existing, non-transactional data
    long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
    long older = now - ttlMillis / 2;
    long newer = now - ttlMillis / 3;
    // timestamps for transactional data
    long nowTx = txVisibilityState.getVisibilityUpperBound();
    long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
    long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;

    Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    ttls.put(familyBytes, ttlMillis);

    List<Cell> cells = new ArrayList<>();
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));

    // Write non-transactional and transactional data
    for (Cell c : cells) {
      region.put(new Put(CellUtil.cloneRow(c)).addColumn(CellUtil.cloneFamily(c), CellUtil.cloneQualifier(c),
                        c.getTimestamp(), CellUtil.cloneValue(c)));
    }

    Scan rawScan = new Scan();
    rawScan.setMaxVersions();

    Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
    Scan txScan = new Scan();
    txScan.setMaxVersions();
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // read all back with raw scanner
    scanAndAssert(region, cells, rawScan);

    // read all back with transaction filter
    scanAndAssert(region, cells, txScan);

    // force a flush to clear the memstore
    region.flushcache(true, false, new FlushLifeCycleTracker() { });
    scanAndAssert(region, cells, txScan);

    // force a major compaction to remove any expired cells
    region.compact(true);
    scanAndAssert(region, cells, txScan);

    // Reduce TTL, this should make cells with timestamps older and olderTx expire
    long newTtl = ttlMillis / 2 - 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // Raw scan should still give all cells
    scanAndAssert(region, cells, rawScan);
    // However, tx scan should not return expired cells
    scanAndAssert(region, select(cells, 1, 3, 5), txScan);

    region.flushcache(true, false, new FlushLifeCycleTracker() { });
    scanAndAssert(region, cells, rawScan);

    // force a major compaction to remove any expired cells
    region.compact(true);
    // This time raw scan too should not return expired cells, as they would be dropped during major compaction
    scanAndAssert(region, select(cells, 1, 3, 5), rawScan);

    // Reduce TTL again to 1 ms, this should expire all cells
    newTtl = 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // force a major compaction to remove expired cells
    region.compact(true);
    // This time raw scan should not return any cells, as all cells have expired.
    scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
  } finally {
    region.close();
  }
}

Example 11

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDeleteFiltering() throws Exception {
  String tableName = "TestDeleteFiltering";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, 0);
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    byte[] row = Bytes.toBytes(1);
    for (int i = 4; i < V.length; i++) {
      Put p = new Put(row);
      p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
      region.put(p);
    }

    // delete from the third entry back
    // take that cell's timestamp + 1 to simulate a delete in a new tx
    long deleteTs = V[5] + 1;
    Delete d = new Delete(row, deleteTs);
    LOG.info("Issuing delete at timestamp " + deleteTs);
    // row deletes are not yet supported (TransactionAwareHTable normally handles this)
    d.deleteColumns(familyBytes, columnBytes);
    region.delete(d);

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, we should drop the deleted version, but not the others
    LOG.info("Flushing region " + region.getRegionNameAsString());
    region.flushcache();

    // now a normal scan should return row with versions at: V[8], V[6].
    // V[7] is invalid and V[5] and prior are deleted.
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);
    // should be only one row
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 1,
                          new long[]{V[8], V[6], deleteTs},
                          new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
  } finally {
    region.close();
  }
}

Example 12

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDataJanitorRegionScanner() throws Exception {
  String tableName = "TestRegionScanner";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    for (int i = 1; i <= 8; i++) {
      for (int k = 1; k <= i; k++) {
        Put p = new Put(Bytes.toBytes(i));
        p.addColumn(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
        region.put(p);
      }
    }

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set

    LOG.info("Flushing region " + region.getRegionInfo().getRegionNameAsString());
    FlushResultImpl flushResult = region.flushcache(true, false, new FlushLifeCycleTracker() { });
    Assert.assertTrue("Unexpected flush result: " + flushResult, flushResult.isFlushSucceeded());

    // now a normal scan should only return the valid rows
    // do not use a filter here to test that cleanup works on flush
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);

    // first returned value should be "4" with version "4"
    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 4, new long[]{V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 5, new long[] {V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 6, new long[]{V[6], V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 7, new long[]{V[6], V[4]});

    results.clear();
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
  } finally {
    region.close();
  }
}

Example 13

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testPreExistingData() throws Exception {
  String tableName = "TestPreExistingData";
  byte[] familyBytes = Bytes.toBytes("f");
  long ttlMillis = TimeUnit.DAYS.toMillis(14);
  HRegion region = createRegion(tableName, familyBytes, ttlMillis);
  try {
    region.initialize();

    // timestamps for pre-existing, non-transactional data
    long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
    long older = now - ttlMillis / 2;
    long newer = now - ttlMillis / 3;
    // timestamps for transactional data
    long nowTx = txVisibilityState.getVisibilityUpperBound();
    long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
    long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;

    Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    ttls.put(familyBytes, ttlMillis);

    List<Cell> cells = new ArrayList<>();
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));

    // Write non-transactional and transactional data
    for (Cell c : cells) {
      region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
    }

    Scan rawScan = new Scan();
    rawScan.setMaxVersions();

    Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
    Scan txScan = new Scan();
    txScan.setMaxVersions();
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // read all back with raw scanner
    scanAndAssert(region, cells, rawScan);

    // read all back with transaction filter
    scanAndAssert(region, cells, txScan);

    // force a flush to clear the memstore
    region.flushcache(true, false);
    scanAndAssert(region, cells, txScan);

    // force a major compaction to remove any expired cells
    region.compact(true);
    scanAndAssert(region, cells, txScan);

    // Reduce TTL, this should make cells with timestamps older and olderTx expire
    long newTtl = ttlMillis / 2 - 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // Raw scan should still give all cells
    scanAndAssert(region, cells, rawScan);
    // However, tx scan should not return expired cells
    scanAndAssert(region, select(cells, 1, 3, 5), txScan);

    region.flushcache(true, false);
    scanAndAssert(region, cells, rawScan);

    // force a major compaction to remove any expired cells
    region.compact(true);
    // This time raw scan too should not return expired cells, as they would be dropped during major compaction
    scanAndAssert(region, select(cells, 1, 3, 5), rawScan);

    // Reduce TTL again to 1 ms, this should expire all cells
    newTtl = 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // force a major compaction to remove expired cells
    region.compact(true);
    // This time raw scan should not return any cells, as all cells have expired.
    scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
  } finally {
    region.close();
  }
}

Example 14

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDeleteFiltering() throws Exception {
  String tableName = "TestDeleteFiltering";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, 0);
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    byte[] row = Bytes.toBytes(1);
    for (int i = 4; i < V.length; i++) {
      Put p = new Put(row);
      p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
      region.put(p);
    }

    // delete from the third entry back
    // take that cell's timestamp + 1 to simulate a delete in a new tx
    long deleteTs = V[5] + 1;
    Delete d = new Delete(row, deleteTs);
    LOG.info("Issuing delete at timestamp " + deleteTs);
    // row deletes are not yet supported (TransactionAwareHTable normally handles this)
    d.deleteColumns(familyBytes, columnBytes);
    region.delete(d);

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, we should drop the deleted version, but not the others
    LOG.info("Flushing region " + region.getRegionInfo().getRegionNameAsString());
    region.flushcache(true, false);

    // now a normal scan should return row with versions at: V[8], V[6].
    // V[7] is invalid and V[5] and prior are deleted.
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);
    // should be only one row
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 1,
        new long[]{V[8], V[6], deleteTs},
        new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
  } finally {
    region.close();
  }
}

Example 15

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDataJanitorRegionScanner() throws Exception {
  String tableName = "TestRegionScanner";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    for (int i = 1; i <= 8; i++) {
      for (int k = 1; k <= i; k++) {
        Put p = new Put(Bytes.toBytes(i));
        p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
        region.put(p);
      }
    }

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set

    LOG.info("Flushing region " + region.getRegionInfo().getRegionNameAsString());
    Region.FlushResult flushResult = region.flushcache(true, false);
    Assert.assertTrue("Unexpected flush result: " + flushResult, flushResult.isFlushSucceeded());

    // now a normal scan should only return the valid rows
    // do not use a filter here to test that cleanup works on flush
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);

    // first returned value should be "4" with version "4"
    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 4, new long[]{V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 5, new long[] {V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 6, new long[]{V[6], V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 7, new long[]{V[6], V[4]});

    results.clear();
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
  } finally {
    region.close();
  }
}

Example 16

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testPreExistingData() throws Exception {
  String tableName = "TestPreExistingData";
  byte[] familyBytes = Bytes.toBytes("f");
  long ttlMillis = TimeUnit.DAYS.toMillis(14);
  HRegion region = createRegion(tableName, familyBytes, ttlMillis);
  try {
    region.initialize();

    // timestamps for pre-existing, non-transactional data
    long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
    long older = now - ttlMillis / 2;
    long newer = now - ttlMillis / 3;
    // timestamps for transactional data
    long nowTx = txVisibilityState.getVisibilityUpperBound();
    long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
    long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;

    Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    ttls.put(familyBytes, ttlMillis);

    List<Cell> cells = new ArrayList<>();
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));

    // Write non-transactional and transactional data
    for (Cell c : cells) {
      region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
    }

    Scan rawScan = new Scan();
    rawScan.setMaxVersions();

    Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
    Scan txScan = new Scan();
    txScan.setMaxVersions();
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // read all back with raw scanner
    scanAndAssert(region, cells, rawScan);

    // read all back with transaction filter
    scanAndAssert(region, cells, txScan);

    // force a flush to clear the memstore
    region.flushcache();
    scanAndAssert(region, cells, txScan);

    // force a major compaction to remove any expired cells
    region.compactStores(true);
    scanAndAssert(region, cells, txScan);

    // Reduce TTL, this should make cells with timestamps older and olderTx expire
    long newTtl = ttlMillis / 2 - 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // Raw scan should still give all cells
    scanAndAssert(region, cells, rawScan);
    // However, tx scan should not return expired cells
    scanAndAssert(region, select(cells, 1, 3, 5), txScan);

    region.flushcache();
    scanAndAssert(region, cells, rawScan);

    // force a major compaction to remove any expired cells
    region.compactStores(true);
    // This time raw scan too should not return expired cells, as they would be dropped during major compaction
    scanAndAssert(region, select(cells, 1, 3, 5), rawScan);

    // Reduce TTL again to 1 ms, this should expire all cells
    newTtl = 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // force a major compaction to remove expired cells
    region.compactStores(true);
    // This time raw scan should not return any cells, as all cells have expired.
    scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
  } finally {
    region.close();
  }
}

Example 17

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testPreExistingData() throws Exception {
  String tableName = "TestPreExistingData";
  byte[] familyBytes = Bytes.toBytes("f");
  long ttlMillis = TimeUnit.DAYS.toMillis(14);
  HRegion region = createRegion(tableName, familyBytes, ttlMillis);
  try {
    region.initialize();

    // timestamps for pre-existing, non-transactional data
    long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
    long older = now - ttlMillis / 2;
    long newer = now - ttlMillis / 3;
    // timestamps for transactional data
    long nowTx = txVisibilityState.getVisibilityUpperBound();
    long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
    long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;

    Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    ttls.put(familyBytes, ttlMillis);

    List<Cell> cells = new ArrayList<>();
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));

    // Write non-transactional and transactional data
    for (Cell c : cells) {
      region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
    }

    Scan rawScan = new Scan();
    rawScan.setMaxVersions();

    Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
    Scan txScan = new Scan();
    txScan.setMaxVersions();
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // read all back with raw scanner
    scanAndAssert(region, cells, rawScan);

    // read all back with transaction filter
    scanAndAssert(region, cells, txScan);

    // force a flush to clear the memstore
    region.flushcache(true, false);
    scanAndAssert(region, cells, txScan);

    // force a major compaction to remove any expired cells
    region.compact(true);
    scanAndAssert(region, cells, txScan);

    // Reduce TTL, this should make cells with timestamps older and olderTx expire
    long newTtl = ttlMillis / 2 - 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // Raw scan should still give all cells
    scanAndAssert(region, cells, rawScan);
    // However, tx scan should not return expired cells
    scanAndAssert(region, select(cells, 1, 3, 5), txScan);

    region.flushcache(true, false);
    scanAndAssert(region, cells, rawScan);

    // force a major compaction to remove any expired cells
    region.compact(true);
    // This time raw scan too should not return expired cells, as they would be dropped during major compaction
    scanAndAssert(region, select(cells, 1, 3, 5), rawScan);

    // Reduce TTL again to 1 ms, this should expire all cells
    newTtl = 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // force a major compaction to remove expired cells
    region.compact(true);
    // This time raw scan should not return any cells, as all cells have expired.
    scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
  } finally {
    region.close();
  }
}

Example 18

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDataJanitorRegionScanner() throws Exception {
  String tableName = "TestRegionScanner";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    for (int i = 1; i <= 8; i++) {
      for (int k = 1; k <= i; k++) {
        Put p = new Put(Bytes.toBytes(i));
        p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
        region.put(p);
      }
    }

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
    LOG.info("Flushing region " + region.getRegionNameAsString());
    region.flushcache(); // in 0.96, there is no indication of success

    // now a normal scan should only return the valid rows - testing that cleanup works on flush
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);

    // first returned value should be "4" with version "4"
    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 4, new long[] {V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 5, new long[] {V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 6, new long[] {V[6], V[4]});

    results.clear();
    assertTrue(regionScanner.next(results));
    assertKeyValueMatches(results, 7, new long[] {V[6], V[4]});

    results.clear();
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
  } finally {
    region.close();
  }
}

Example 19

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testPreExistingData() throws Exception {
  String tableName = "TestPreExistingData";
  byte[] familyBytes = Bytes.toBytes("f");
  long ttlMillis = TimeUnit.DAYS.toMillis(14);
  HRegion region = createRegion(tableName, familyBytes, ttlMillis);
  try {
    region.initialize();

    // timestamps for pre-existing, non-transactional data
    long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
    long older = now - ttlMillis / 2;
    long newer = now - ttlMillis / 3;
    // timestamps for transactional data
    long nowTx = txVisibilityState.getVisibilityUpperBound();
    long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
    long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;

    Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    ttls.put(familyBytes, ttlMillis);

    List<Cell> cells = new ArrayList<>();
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
    cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
    cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
    cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));

    // Write non-transactional and transactional data
    for (Cell c : cells) {
      region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
    }

    Scan rawScan = new Scan();
    rawScan.setMaxVersions();

    Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
    Scan txScan = new Scan();
    txScan.setMaxVersions();
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // read all back with raw scanner
    scanAndAssert(region, cells, rawScan);

    // read all back with transaction filter
    scanAndAssert(region, cells, txScan);

    // force a flush to clear the memstore
    region.flushcache(true, false);
    scanAndAssert(region, cells, txScan);

    // force a major compaction to remove any expired cells
    region.compact(true);
    scanAndAssert(region, cells, txScan);

    // Reduce TTL, this should make cells with timestamps older and olderTx expire
    long newTtl = ttlMillis / 2 - 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // Raw scan should still give all cells
    scanAndAssert(region, cells, rawScan);
    // However, tx scan should not return expired cells
    scanAndAssert(region, select(cells, 1, 3, 5), txScan);

    region.flushcache(true, false);
    scanAndAssert(region, cells, rawScan);

    // force a major compaction to remove any expired cells
    region.compact(true);
    // This time raw scan too should not return expired cells, as they would be dropped during major compaction
    scanAndAssert(region, select(cells, 1, 3, 5), rawScan);

    // Reduce TTL again to 1 ms, this should expire all cells
    newTtl = 1;
    region = updateTtl(region, familyBytes, newTtl);
    ttls.put(familyBytes, newTtl);
    txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
                        TxUtils.getMaxVisibleTimestamp(dummyTransaction));
    txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));

    // force a major compaction to remove expired cells
    region.compact(true);
    // This time raw scan should not return any cells, as all cells have expired.
    scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
  } finally {
    region.close();
  }
}

Example 20

Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0

4 votes

@Test
public void testDeleteFiltering() throws Exception {
  String tableName = "TestDeleteFiltering";
  byte[] familyBytes = Bytes.toBytes("f");
  byte[] columnBytes = Bytes.toBytes("c");
  HRegion region = createRegion(tableName, familyBytes, 0);
  try {
    region.initialize();
    TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
    LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache));

    byte[] row = Bytes.toBytes(1);
    for (int i = 4; i < V.length; i++) {
      Put p = new Put(row);
      p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
      region.put(p);
    }

    // delete from the third entry back
    // take that cell's timestamp + 1 to simulate a delete in a new tx
    long deleteTs = V[5] + 1;
    Delete d = new Delete(row, deleteTs);
    LOG.info("Issuing delete at timestamp " + deleteTs);
    // row deletes are not yet supported (TransactionAwareHTable normally handles this)
    d.deleteColumns(familyBytes, columnBytes);
    region.delete(d);

    List<Cell> results = Lists.newArrayList();

    // force a flush to clear the data
    // during flush, we should drop the deleted version, but not the others
    LOG.info("Flushing region " + region.getRegionInfo().getRegionNameAsString());
    region.flushcache(true, false);

    // now a normal scan should return row with versions at: V[8], V[6].
    // V[7] is invalid and V[5] and prior are deleted.
    Scan scan = new Scan();
    scan.setMaxVersions(10);
    RegionScanner regionScanner = region.getScanner(scan);
    // should be only one row
    assertFalse(regionScanner.next(results));
    assertKeyValueMatches(results, 1,
        new long[]{V[8], V[6], deleteTs},
        new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
  } finally {
    region.close();
  }
}