Java Code Examples for org.apache.hadoop.hbase.KeyValueUtil#ensureKeyValue()

The following examples show how to use org.apache.hadoop.hbase.KeyValueUtil#ensureKeyValue() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TTable.java    From phoenix-omid with Apache License 2.0 6 votes vote down vote up
/**
 * @param put an instance of Put
 * @param timestamp  timestamp to be used as cells version
 * @param commitTimestamp  timestamp to be used as commit timestamp
 * @throws IOException if a remote or network exception occurs.
 */
static public Put markPutAsCommitted(Put put, long timestamp, long commitTimestamp) {
    final Put tsput = new Put(put.getRow(), timestamp);
    propagateAttributes(put, tsput);

    Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
    for (List<Cell> kvl : kvs.values()) {
        for (Cell c : kvl) {
            KeyValue kv = KeyValueUtil.ensureKeyValue(c);
            Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), timestamp);
            try {
                tsput.add(kv);
            } catch (IOException e) {
                // The existing Put has this Cell, so the cloned one
                // will never throw an IOException when it's added.
                throw new RuntimeException(e);
            }
            tsput.addColumn(CellUtil.cloneFamily(kv),
                    CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                    kv.getTimestamp(),
                    Bytes.toBytes(commitTimestamp));
        }
    }

    return tsput;
}
 
Example 2
Source File: IndexMemStore.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * Move forward on the sub-lists set previously by seek.
 * @param key seek value (should be non-null)
 * @return true if there is at least one KV to read, false otherwise
 */
@Override
public synchronized boolean reseek(Cell key) {
  /*
   * See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation. This
   * code is executed concurrently with flush and puts, without locks. Two points must be known
   * when working on this code: 1) It's not possible to use the 'kvTail' and 'snapshot'
   * variables, as they are modified during a flush. 2) The ideal implementation for performance
   * would use the sub skip list implicitly pointed by the iterators 'kvsetIt' and 'snapshotIt'.
   * Unfortunately the Java API does not offer a method to get it. So we remember the last keys
   * we iterated to and restore the reseeked set to at least that point.
   */

  KeyValue kv = KeyValueUtil.ensureKeyValue(key);
  kvsetIt = kvsetAtCreation.tailSet(getHighest(kv, kvsetItRow)).iterator();
  return seekInSubLists();
}
 
Example 3
Source File: TestVisibilityLabelsReplication.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put m, WALEdit edit,
    Durability durability) throws IOException {
  byte[] attribute = m.getAttribute(NON_VISIBILITY);
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<>();
  if (attribute != null) {
    for (List<? extends Cell> edits : m.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = CellUtil.cloneFamily(kv);
        }
        Tag tag = new ArrayBackedTag((byte) NON_VIS_TAG_TYPE, attribute);
        List<Tag> tagList = new ArrayList<>(PrivateCellUtil.getTags(cell).size() + 1);
        tagList.add(tag);
        tagList.addAll(PrivateCellUtil.getTags(cell));
        Cell newcell = PrivateCellUtil.createCell(kv, tagList);
        ((List<Cell>) updatedCells).add(newcell);
      }
    }
    m.getFamilyCellMap().remove(cf);
    // Update the family map
    m.getFamilyCellMap().put(cf, updatedCells);
  }
}
 
Example 4
Source File: TTable.java    From phoenix-omid with Apache License 2.0 5 votes vote down vote up
private Put putInternal(Transaction tx, Put put, boolean addShadowCell) throws IOException {

        throwExceptionIfOpSetsTimerange(put);

        HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);

        final long writeTimestamp = transaction.getWriteTimestamp();

        // create put with correct ts
        final Put tsput = new Put(put.getRow(), writeTimestamp);
        propagateAttributes(put, tsput);
        Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
        for (List<Cell> kvl : kvs.values()) {
            for (Cell c : kvl) {
                CellUtils.validateCell(c, writeTimestamp);
                // Reach into keyvalue to update timestamp.
                // It's not nice to reach into keyvalue internals,
                // but we want to avoid having to copy the whole thing
                KeyValue kv = KeyValueUtil.ensureKeyValue(c);
                Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), writeTimestamp);
                tsput.add(kv);

                if (addShadowCell) {
                    tsput.addColumn(CellUtil.cloneFamily(kv),
                            CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                            kv.getTimestamp(),
                            Bytes.toBytes(kv.getTimestamp()));
                } else {
                    HBaseCellId cellId = new HBaseCellId(this,
                            CellUtil.cloneRow(kv),
                            CellUtil.cloneFamily(kv),
                            CellUtil.cloneQualifier(kv),
                            kv.getTimestamp());

                    addWriteSetElement(transaction, cellId);
                }
            }
        }
        return tsput;
    }
 
Example 5
Source File: LocalTableStateTest.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * Test that we correctly rollback the state of keyvalue
 * @throws Exception
 */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
  Put m = new Put(row);
  m.addColumn(fam, qual, ts, val);
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final byte[] stored = Bytes.toBytes("stored-value");
  final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  storedKv.setSequenceId(2);

  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)storedKv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, m);

  // add the kvs from the mutation
  KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
  kv.setSequenceId(0);
  table.addPendingUpdates(kv);

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", kv, s.next());

  // rollback that value
  table.rollback(Arrays.asList(kv));
  p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  s = p.getFirst();
  assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
}
 
Example 6
Source File: TestFilterList.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Tests the behavior of transform() in a hierarchical filter.
 *
 * transform() only applies after a filterCell() whose return-code includes the KeyValue.
 * Lazy evaluation of AND
 */
@Test
public void testTransformMPO() throws Exception {
  // Apply the following filter:
  //     (family=fam AND qualifier=qual1 AND KeyOnlyFilter)
  //  OR (family=fam AND qualifier=qual2)
  final FilterList flist = new FilterList(Operator.MUST_PASS_ONE, Lists.<Filter>newArrayList(
      new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
          new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
          new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("qual1"))),
          new KeyOnlyFilter())),
      new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
          new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
          new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("qual2")))))));

  final KeyValue kvQual1 = new KeyValue(
      Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual1"), Bytes.toBytes("value"));
  final KeyValue kvQual2 = new KeyValue(
      Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual2"), Bytes.toBytes("value"));
  final KeyValue kvQual3 = new KeyValue(
      Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual3"), Bytes.toBytes("value"));

  // Value for fam:qual1 should be stripped:
  assertEquals(Filter.ReturnCode.INCLUDE, flist.filterCell(kvQual1));
  final KeyValue transformedQual1 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual1));
  assertEquals(0, transformedQual1.getValueLength());

  // Value for fam:qual2 should not be stripped:
  assertEquals(Filter.ReturnCode.INCLUDE, flist.filterCell(kvQual2));
  final KeyValue transformedQual2 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual2));
  assertEquals("value", Bytes.toString(transformedQual2.getValueArray(),
    transformedQual2.getValueOffset(), transformedQual2.getValueLength()));

  // Other keys should be skipped:
  assertEquals(Filter.ReturnCode.SKIP, flist.filterCell(kvQual3));
}
 
Example 7
Source File: TestHRegionServerBulkLoad.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) {
  for (Cell cell : logEdit.getCells()) {
    KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
    for (Map.Entry entry : kv.toStringMap().entrySet()) {
      if (entry.getValue().equals(Bytes.toString(WALEdit.BULK_LOAD))) {
        found = true;
      }
    }
  }
}
 
Example 8
Source File: TestTags.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void updateMutationAddingTags(final Mutation m) {
  byte[] attribute = m.getAttribute("visibility");
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<>();
  if (attribute != null) {
    for (List<? extends Cell> edits : m.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = CellUtil.cloneFamily(kv);
        }
        Tag tag = new ArrayBackedTag((byte) 1, attribute);
        List<Tag> tagList = new ArrayList<>();
        tagList.add(tag);

        KeyValue newKV = new KeyValue(CellUtil.cloneRow(kv), 0, kv.getRowLength(),
            CellUtil.cloneFamily(kv), 0, kv.getFamilyLength(), CellUtil.cloneQualifier(kv), 0,
            kv.getQualifierLength(), kv.getTimestamp(),
            KeyValue.Type.codeToType(kv.getTypeByte()), CellUtil.cloneValue(kv), 0,
            kv.getValueLength(), tagList);
        ((List<Cell>) updatedCells).add(newKV);
      }
    }
    m.getFamilyCellMap().remove(cf);
    // Update the family map
    m.getFamilyCellMap().put(cf, updatedCells);
  }
}
 
Example 9
Source File: TestReplicationWithTags.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e, final Put put,
    final WALEdit edit, final Durability durability) throws IOException {
  byte[] attribute = put.getAttribute("visibility");
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<>();
  if (attribute != null) {
    for (List<? extends Cell> edits : put.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = CellUtil.cloneFamily(kv);
        }
        Tag tag = new ArrayBackedTag(TAG_TYPE, attribute);
        List<Tag> tagList = new ArrayList<>(1);
        tagList.add(tag);

        KeyValue newKV = new KeyValue(CellUtil.cloneRow(kv), 0, kv.getRowLength(),
            CellUtil.cloneFamily(kv), 0, kv.getFamilyLength(), CellUtil.cloneQualifier(kv), 0,
            kv.getQualifierLength(), kv.getTimestamp(),
            KeyValue.Type.codeToType(kv.getTypeByte()), CellUtil.cloneValue(kv), 0,
            kv.getValueLength(), tagList);
        ((List<Cell>) updatedCells).add(newKV);
      }
    }
    put.getFamilyCellMap().remove(cf);
    // Update the family map
    put.getFamilyCellMap().put(cf, updatedCells);
  }
}
 
Example 10
Source File: SortMergeJoinPlan.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Override
protected void writeToBuffer(MappedByteBuffer buffer, Tuple e) {
    KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
    buffer.putInt(kv.getLength() + Bytes.SIZEOF_INT);
    buffer.putInt(kv.getLength());
    buffer.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
}
 
Example 11
Source File: TestLocalTableState.java    From phoenix with Apache License 2.0 4 votes vote down vote up
/**
 * Test that we correctly rollback the state of keyvalue
 * @throws Exception
 */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
  Put m = new Put(row);
  m.add(fam, qual, ts, val);
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  HRegion region = Mockito.mock(HRegion.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  RegionScanner scanner = Mockito.mock(RegionScanner.class);
  Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
  final byte[] stored = Bytes.toBytes("stored-value");
  final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  storedKv.setSequenceId(2);
  Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];

      list.add(storedKv);
      return false;
    }
  });
  LocalHBaseState state = new LocalTable(env);
  LocalTableState table = new LocalTableState(env, state, m);
  // add the kvs from the mutation
  KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
  kv.setSequenceId(0);
  table.addPendingUpdates(kv);

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col));
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", kv, s.next());

  // rollback that value
  table.rollback(Arrays.asList(kv));
  p = table.getIndexedColumnsTableState(Arrays.asList(col));
  s = p.getFirst();
  assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
  Mockito.verify(env, Mockito.times(1)).getRegion();
  Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class));
}
 
Example 12
Source File: StatisticsCollector.java    From phoenix with Apache License 2.0 4 votes vote down vote up
/**
 * Update the current statistics based on the latest batch of key-values from the underlying scanner
 * 
 * @param results
 *            next batch of {@link KeyValue}s
 */
public void collectStatistics(final List<Cell> results) {
    Map<ImmutableBytesPtr, Boolean> famMap = Maps.newHashMap();
    List<GuidePostsInfo> rowTracker = null;
    if(cachedGps == null) {
        rowTracker = 
                new ArrayList<GuidePostsInfo>();
    }
    for (Cell cell : results) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp());
        Pair<Long, GuidePostsInfo> gps;
        if (cachedGps == null) {
            ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(),
                    kv.getFamilyLength());
            gps = guidePostsMap.get(cfKey);
            if (gps == null) {
                gps = new Pair<Long, GuidePostsInfo>(0l, new GuidePostsInfo(0,
                        Collections.<byte[]> emptyList(), 0l));
                guidePostsMap.put(cfKey, gps);
            }
            if (famMap.get(cfKey) == null) {
                famMap.put(cfKey, true);
                rowTracker.add(gps.getSecond());
            }
        } else {
            gps = cachedGps;
        }
        int kvLength = kv.getLength();
        long byteCount = gps.getFirst() + kvLength;
        gps.setFirst(byteCount);
        if (byteCount >= guidepostDepth) {
            byte[] row = ByteUtil.copyKeyBytesIfNecessary(new ImmutableBytesWritable(kv.getRowArray(), kv
                    .getRowOffset(), kv.getRowLength()));
            if (gps.getSecond().addGuidePost(row, byteCount)) {
                gps.setFirst(0l);
            }
        }
    }
    if(cachedGps == null) {
        for (GuidePostsInfo s : rowTracker) {
            s.incrementRowCount();
        }
    } else {
        cachedGps.getSecond().incrementRowCount();
    }
}
 
Example 13
Source File: DataBlockEncodingTool.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Check statistics for given HFile for different data block encoders.
 * @param scanner Of file which will be compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @throws IOException thrown if scanner is invalid
 */
public void checkStatistics(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  scanner.seek(KeyValue.LOWESTKEY);

  KeyValue currentKV;

  byte[] previousKey = null;
  byte[] currentKey;

  DataBlockEncoding[] encodings = DataBlockEncoding.values();

  ByteArrayOutputStream uncompressedOutputStream =
      new ByteArrayOutputStream();

  int j = 0;
  while ((currentKV = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
    // Iterates through key/value pairs
    j++;
    currentKey = currentKV.getKey();
    if (previousKey != null) {
      for (int i = 0; i < previousKey.length && i < currentKey.length &&
          previousKey[i] == currentKey[i]; ++i) {
        totalKeyRedundancyLength++;
      }
    }

    // Add tagsLen zero to cells don't include tags. Since the process of
    // scanner converts byte array to KV would abandon tagsLen part if tagsLen
    // is zero. But we still needs the tagsLen part to check if current cell
    // include tags. If USE_TAG is true, HFile contains cells with tags,
    // if the cell tagsLen equals 0, it means other cells may have tags.
    if (USE_TAG && currentKV.getTagsLength() == 0) {
      uncompressedOutputStream.write(currentKV.getBuffer(),
          currentKV.getOffset(), currentKV.getLength());
      // write tagsLen = 0.
      uncompressedOutputStream.write(Bytes.toBytes((short) 0));
    } else {
      uncompressedOutputStream.write(currentKV.getBuffer(),
          currentKV.getOffset(), currentKV.getLength());
    }

    if(includesMemstoreTS) {
      WritableUtils.writeVLong(
          new DataOutputStream(uncompressedOutputStream), currentKV.getSequenceId());
    }

    previousKey = currentKey;

    int kLen = currentKV.getKeyLength();
    int vLen = currentKV.getValueLength();
    int cfLen = currentKV.getFamilyLength(currentKV.getFamilyOffset());
    int restLen = currentKV.getLength() - kLen - vLen;

    totalKeyLength += kLen;
    totalValueLength += vLen;
    totalPrefixLength += restLen;
    totalCFLength += cfLen;
  }

  rawKVs = uncompressedOutputStream.toByteArray();
  for (DataBlockEncoding encoding : encodings) {
    if (encoding == DataBlockEncoding.NONE) {
      continue;
    }
    DataBlockEncoder d = encoding.getEncoder();
    HFileContext meta = new HFileContextBuilder()
        .withDataBlockEncoding(encoding)
        .withCompression(Compression.Algorithm.NONE)
        .withIncludesMvcc(includesMemstoreTS)
        .withIncludesTags(USE_TAG).build();
    codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta ));
  }
}
 
Example 14
Source File: DataBlockEncodingTool.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Verify if all data block encoders are working properly.
 *
 * @param scanner Of file which was compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @return true if all data block encoders compressed/decompressed correctly.
 * @throws IOException thrown if scanner is invalid
 */
public boolean verifyCodecs(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  KeyValue currentKv;

  scanner.seek(KeyValue.LOWESTKEY);
  List<Iterator<Cell>> codecIterators = new ArrayList<>();
  for(EncodedDataBlock codec : codecs) {
    codecIterators.add(codec.getIterator(HFileBlock.headerSize(useHBaseChecksum)));
  }

  int j = 0;
  while ((currentKv = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
    // Iterates through key/value pairs
    ++j;
    for (Iterator<Cell> it : codecIterators) {
      Cell c = it.next();
      KeyValue codecKv = KeyValueUtil.ensureKeyValue(c);
      if (codecKv == null || 0 != Bytes.compareTo(
          codecKv.getBuffer(), codecKv.getOffset(), codecKv.getLength(),
          currentKv.getBuffer(), currentKv.getOffset(),
          currentKv.getLength())) {
        if (codecKv == null) {
          LOG.error("There is a bug in codec " + it +
              " it returned null KeyValue,");
        } else {
          int prefix = 0;
          int limitLength = 2 * Bytes.SIZEOF_INT +
              Math.min(codecKv.getLength(), currentKv.getLength());
          while (prefix < limitLength &&
              codecKv.getBuffer()[prefix + codecKv.getOffset()] ==
              currentKv.getBuffer()[prefix + currentKv.getOffset()]) {
            prefix++;
          }

          LOG.error("There is bug in codec " + it.toString() +
              "\n on element " + j +
              "\n codecKv.getKeyLength() " + codecKv.getKeyLength() +
              "\n codecKv.getValueLength() " + codecKv.getValueLength() +
              "\n codecKv.getLength() " + codecKv.getLength() +
              "\n currentKv.getKeyLength() " + currentKv.getKeyLength() +
              "\n currentKv.getValueLength() " + currentKv.getValueLength() +
              "\n codecKv.getLength() " + currentKv.getLength() +
              "\n currentKV rowLength " + currentKv.getRowLength() +
              " familyName " + currentKv.getFamilyLength() +
              " qualifier " + currentKv.getQualifierLength() +
              "\n prefix " + prefix +
              "\n codecKv   '" + Bytes.toStringBinary(codecKv.getBuffer(),
                  codecKv.getOffset(), prefix) + "' diff '" +
                  Bytes.toStringBinary(codecKv.getBuffer(),
                      codecKv.getOffset() + prefix, codecKv.getLength() -
                      prefix) + "'" +
              "\n currentKv '" + Bytes.toStringBinary(
                 currentKv.getBuffer(),
                 currentKv.getOffset(), prefix) + "' diff '" +
                 Bytes.toStringBinary(currentKv.getBuffer(),
                     currentKv.getOffset() + prefix, currentKv.getLength() -
                     prefix) + "'"
              );
        }
        return false;
      }
    }
  }

  LOG.info("Verification was successful!");

  return true;
}
 
Example 15
Source File: SortMergeJoinPlan.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Override
protected int sizeOf(Tuple e) {
    KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
    return Bytes.SIZEOF_INT * 2 + kv.getLength();
}
 
Example 16
Source File: PutCombiner.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
protected void reduce(K row, Iterable<Put> vals, Context context)
    throws IOException, InterruptedException {
  // Using HeapSize to create an upper bound on the memory size of
  // the puts and flush some portion of the content while looping. This
  // flush could result in multiple Puts for a single rowkey. That is
  // acceptable because Combiner is run as an optimization and it's not
  // critical that all Puts are grouped perfectly.
  long threshold = context.getConfiguration().getLong(
      "putcombiner.row.threshold", 1L * (1<<30));
  int cnt = 0;
  long curSize = 0;
  Put put = null;
  Map<byte[], List<Cell>> familyMap = null;
  for (Put p : vals) {
    cnt++;
    if (put == null) {
      put = p;
      familyMap = put.getFamilyCellMap();
    } else {
      for (Entry<byte[], List<Cell>> entry : p.getFamilyCellMap()
          .entrySet()) {
        List<Cell> cells = familyMap.get(entry.getKey());
        List<Cell> kvs = (cells != null) ? (List<Cell>) cells : null;
        for (Cell cell : entry.getValue()) {
          KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
          curSize += kv.heapSize();
          if (kvs != null) {
            kvs.add(kv);
          }
        }
        if (cells == null) {
          familyMap.put(entry.getKey(), entry.getValue());
        }
      }
      if (cnt % 10 == 0) context.setStatus("Combine " + cnt);
      if (curSize > threshold) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format("Combined %d Put(s) into %d.", cnt, 1));
        }
        context.write(row, put);
        put = null;
        curSize = 0;
        cnt = 0;
      }
    }
  }
  if (put != null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(String.format("Combined %d Put(s) into %d.", cnt, 1));
    }
    context.write(row, put);
  }
}
 
Example 17
Source File: ApplyAndFilterDeletesFilter.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Override
public ReturnCode filterKeyValue(Cell next) {
  // we marked ourselves done, but the END_ROW_KEY didn't manage to seek to the very last key
  if (this.done) {
    return ReturnCode.SKIP;
  }

  KeyValue nextKV = KeyValueUtil.ensureKeyValue(next);
  switch (KeyValue.Type.codeToType(next.getTypeByte())) {
  /*
   * DeleteFamily will always sort first because those KVs (we assume) don't have qualifiers (or
   * rather are null). Therefore, we have to keep a hold of all the delete families until we get
   * to a Put entry that is covered by that delete (in which case, we are done with the family).
   */
  case DeleteFamily:
    // track the family to delete. If we are updating the delete, that means we have passed all
    // kvs in the last column, so we can safely ignore the last deleteFamily, and just use this
    // one. In fact, it means that all the previous deletes can be ignored because the family must
    // not match anymore.
    this.coveringDelete.reset();
    this.coveringDelete.deleteFamily = nextKV;
    return ReturnCode.SKIP;
  case DeleteColumn:
    // similar to deleteFamily, all the newer deletes/puts would have been seen at this point, so
    // we can safely replace the more recent delete column with the more recent one
    this.coveringDelete.pointDelete = null;
    this.coveringDelete.deleteColumn = nextKV;
    return ReturnCode.SKIP;
  case Delete:
    // we are just deleting the single column value at this point.
    // therefore we just skip this entry and go onto the next one. The only caveat is that
    // we should still cover the next entry if this delete applies to the next entry, so we
    // have to keep around a reference to the KV to compare against the next valid entry
    this.coveringDelete.pointDelete = nextKV;
    return ReturnCode.SKIP;
  default:
    // no covering deletes
    if (coveringDelete.empty()) {
      return ReturnCode.INCLUDE;
    }

    if (coveringDelete.matchesFamily(nextKV)) {
      this.currentHint = familyHint;
      return ReturnCode.SEEK_NEXT_USING_HINT;
    }

    if (coveringDelete.matchesColumn(nextKV)) {
      // hint to the next column
      this.currentHint = columnHint;
      return ReturnCode.SEEK_NEXT_USING_HINT;
    }

    if (coveringDelete.matchesPoint(nextKV)) {
      return ReturnCode.SKIP;
    }

  }

  // none of the deletes matches, we are done
  return ReturnCode.INCLUDE;
}
 
Example 18
Source File: SepConsumer.java    From hbase-indexer with Apache License 2.0 4 votes vote down vote up
@Override
public AdminProtos.ReplicateWALEntryResponse replicateWALEntry(final RpcController controller,
        final AdminProtos.ReplicateWALEntryRequest request) throws ServiceException {
    try {

        // TODO Recording of last processed timestamp won't work if two batches of log entries are sent out of order
        long lastProcessedTimestamp = -1;

        SepEventExecutor eventExecutor = new SepEventExecutor(listener, executors, 100, sepMetrics);

        List<AdminProtos.WALEntry> entries = request.getEntryList();
        CellScanner cells = ((PayloadCarryingRpcController)controller).cellScanner();

        for (final AdminProtos.WALEntry entry : entries) {
            TableName tableName = (entry.getKey().getWriteTime() < subscriptionTimestamp) ? null :
                    TableName.valueOf(entry.getKey().getTableName().toByteArray());
            Multimap<ByteBuffer, Cell> keyValuesPerRowKey = ArrayListMultimap.create();
            final Map<ByteBuffer, byte[]> payloadPerRowKey = Maps.newHashMap();
            int count = entry.getAssociatedCellCount();
            for (int i = 0; i < count; i++) {
                if (!cells.advance()) {
                    throw new ArrayIndexOutOfBoundsException("Expected=" + count + ", index=" + i);
                }

                // this signals to us that we simply need to skip over count of cells
                if (tableName == null) {
                    continue;
                }

                Cell cell = cells.current();
                ByteBuffer rowKey = ByteBuffer.wrap(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
                byte[] payload;
                KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
                if (payloadExtractor != null && (payload = payloadExtractor.extractPayload(tableName.toBytes(), kv)) != null) {
                    if (payloadPerRowKey.containsKey(rowKey)) {
                        log.error("Multiple payloads encountered for row " + Bytes.toStringBinary(rowKey)
                                + ", choosing " + Bytes.toStringBinary(payloadPerRowKey.get(rowKey)));
                    } else {
                        payloadPerRowKey.put(rowKey, payload);
                    }
                }
                keyValuesPerRowKey.put(rowKey, kv);
            }

            for (final ByteBuffer rowKeyBuffer : keyValuesPerRowKey.keySet()) {
                final List<Cell> keyValues = (List<Cell>) keyValuesPerRowKey.get(rowKeyBuffer);

                final SepEvent sepEvent = new SepEvent(tableName.toBytes(), CellUtil.cloneRow(keyValues.get(0)), keyValues,
                        payloadPerRowKey.get(rowKeyBuffer));
                eventExecutor.scheduleSepEvent(sepEvent);
                lastProcessedTimestamp = Math.max(lastProcessedTimestamp, entry.getKey().getWriteTime());
            }

        }
        List<Future<?>> futures = eventExecutor.flush();
        waitOnSepEventCompletion(futures);

        if (lastProcessedTimestamp > 0) {
            sepMetrics.reportSepTimestamp(lastProcessedTimestamp);
        }
        return AdminProtos.ReplicateWALEntryResponse.newBuilder().build();
    } catch (IOException ie) {
        throw new ServiceException(ie);
    }
}