Java Code Examples for org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil#toScan()

private Scan validateKey(final RegionInfo region, final ExportProtos.ExportRequest request)
    throws IOException {
  Scan scan = ProtobufUtil.toScan(request.getScan());
  byte[] regionStartKey = region.getStartKey();
  byte[] originStartKey = scan.getStartRow();
  if (originStartKey == null
          || Bytes.compareTo(originStartKey, regionStartKey) < 0) {
    scan.withStartRow(regionStartKey);
  }
  byte[] regionEndKey = region.getEndKey();
  byte[] originEndKey = scan.getStopRow();
  if (originEndKey == null
          || Bytes.compareTo(originEndKey, regionEndKey) > 0) {
    scan.withStartRow(regionEndKey);
  }
  return scan;
}
 

@Test
public void testAttributesSerialization() throws IOException {
  Scan scan = new Scan();
  scan.setAttribute("attribute1", Bytes.toBytes("value1"));
  scan.setAttribute("attribute2", Bytes.toBytes("value2"));
  scan.setAttribute("attribute3", Bytes.toBytes("value3"));

  ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan);

  Scan scan2 = ProtobufUtil.toScan(scanProto);

  Assert.assertNull(scan2.getAttribute("absent"));
  Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan2.getAttribute("attribute1")));
  Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan2.getAttribute("attribute2")));
  Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), scan2.getAttribute("attribute3")));
  Assert.assertEquals(3, scan2.getAttributesMap().size());
}
 

private RegionScannerHolder newRegionScanner(ScanRequest request, ScanResponse.Builder builder)
    throws IOException {
  HRegion region = getRegion(request.getRegion());
  rejectIfInStandByState(region);
  ClientProtos.Scan protoScan = request.getScan();
  boolean isLoadingCfsOnDemandSet = protoScan.hasLoadColumnFamiliesOnDemand();
  Scan scan = ProtobufUtil.toScan(protoScan);
  // if the request doesn't set this, get the default region setting.
  if (!isLoadingCfsOnDemandSet) {
    scan.setLoadColumnFamiliesOnDemand(region.isLoadingCfsOnDemandDefault());
  }

  if (!scan.hasFamilies()) {
    // Adding all families to scanner
    for (byte[] family : region.getTableDescriptor().getColumnFamilyNames()) {
      scan.addFamily(family);
    }
  }
  if (region.getCoprocessorHost() != null) {
    // preScannerOpen is not allowed to return a RegionScanner. Only post hook can create a
    // wrapper for the core created RegionScanner
    region.getCoprocessorHost().preScannerOpen(scan);
  }
  RegionScannerImpl coreScanner = region.getScanner(scan);
  Shipper shipper = coreScanner;
  RegionScanner scanner = coreScanner;
  if (region.getCoprocessorHost() != null) {
    scanner = region.getCoprocessorHost().postScannerOpen(scan, scanner);
  }
  long scannerId = scannerIdGenerator.generateNewScannerId();
  builder.setScannerId(scannerId);
  builder.setMvccReadPoint(scanner.getMvccReadPoint());
  builder.setTtl(scannerLeaseTimeoutPeriod);
  String scannerName = String.valueOf(scannerId);
  return addScanner(scannerName, scanner, shipper, region, scan.isNeedCursorResult());
}
 

/**
 * Gives the maximum for a given combination of column qualifier and column
 * family, in the given row range as defined in the Scan object. In its
 * current implementation, it takes one column family and one column qualifier
 * (if provided). In case of null column qualifier, maximum value for the
 * entire column family will be returned.
 */
@Override
public void getMax(RpcController controller, AggregateRequest request,
        RpcCallback<AggregateResponse> done) {
  InternalScanner scanner = null;
  AggregateResponse response = null;
  T max = null;
  try {
    ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request);
    T temp;
    Scan scan = ProtobufUtil.toScan(request.getScan());
    scanner = env.getRegion().getScanner(scan);
    List<Cell> results = new ArrayList<>();
    byte[] colFamily = scan.getFamilies()[0];
    NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
    byte[] qualifier = null;
    if (qualifiers != null && !qualifiers.isEmpty()) {
      qualifier = qualifiers.pollFirst();
    }
    // qualifier can be null.
    boolean hasMoreRows = false;
    do {
      hasMoreRows = scanner.next(results);
      int listSize = results.size();
      for (int i = 0; i < listSize; i++) {
        temp = ci.getValue(colFamily, qualifier, results.get(i));
        max = (max == null || (temp != null && ci.compare(temp, max) > 0)) ? temp : max;
      }
      results.clear();
    } while (hasMoreRows);
    if (max != null) {
      AggregateResponse.Builder builder = AggregateResponse.newBuilder();
      builder.addFirstPart(ci.getProtoForCellType(max).toByteString());
      response = builder.build();
    }
  } catch (IOException e) {
    CoprocessorRpcUtils.setControllerException(controller, e);
  } finally {
    if (scanner != null) {
      try {
        scanner.close();
      } catch (IOException ignored) {}
    }
  }
  log.info("Maximum from this region is "
      + env.getRegion().getRegionInfo().getRegionNameAsString() + ": " + max);
  done.run(response);
}
 

/**
 * Gives the minimum for a given combination of column qualifier and column
 * family, in the given row range as defined in the Scan object. In its
 * current implementation, it takes one column family and one column qualifier
 * (if provided). In case of null column qualifier, minimum value for the
 * entire column family will be returned.
 */
@Override
public void getMin(RpcController controller, AggregateRequest request,
        RpcCallback<AggregateResponse> done) {
  AggregateResponse response = null;
  InternalScanner scanner = null;
  T min = null;
  try {
    ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request);
    T temp;
    Scan scan = ProtobufUtil.toScan(request.getScan());
    scanner = env.getRegion().getScanner(scan);
    List<Cell> results = new ArrayList<>();
    byte[] colFamily = scan.getFamilies()[0];
    NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
    byte[] qualifier = null;
    if (qualifiers != null && !qualifiers.isEmpty()) {
      qualifier = qualifiers.pollFirst();
    }
    boolean hasMoreRows = false;
    do {
      hasMoreRows = scanner.next(results);
      int listSize = results.size();
      for (int i = 0; i < listSize; i++) {
        temp = ci.getValue(colFamily, qualifier, results.get(i));
        min = (min == null || (temp != null && ci.compare(temp, min) < 0)) ? temp : min;
      }
      results.clear();
    } while (hasMoreRows);
    if (min != null) {
      response = AggregateResponse.newBuilder().addFirstPart(
        ci.getProtoForCellType(min).toByteString()).build();
    }
  } catch (IOException e) {
    CoprocessorRpcUtils.setControllerException(controller, e);
  } finally {
    if (scanner != null) {
      try {
        scanner.close();
      } catch (IOException ignored) {}
    }
  }
  log.info("Minimum from this region is "
      + env.getRegion().getRegionInfo().getRegionNameAsString() + ": " + min);
  done.run(response);
}
 

/**
 * Gives the sum for a given combination of column qualifier and column
 * family, in the given row range as defined in the Scan object. In its
 * current implementation, it takes one column family and one column qualifier
 * (if provided). In case of null column qualifier, sum for the entire column
 * family will be returned.
 */
@Override
public void getSum(RpcController controller, AggregateRequest request,
        RpcCallback<AggregateResponse> done) {
  AggregateResponse response = null;
  InternalScanner scanner = null;
  long sum = 0L;
  try {
    ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request);
    S sumVal = null;
    T temp;
    Scan scan = ProtobufUtil.toScan(request.getScan());
    scanner = env.getRegion().getScanner(scan);
    byte[] colFamily = scan.getFamilies()[0];
    NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
    byte[] qualifier = null;
    if (qualifiers != null && !qualifiers.isEmpty()) {
      qualifier = qualifiers.pollFirst();
    }
    List<Cell> results = new ArrayList<>();
    boolean hasMoreRows = false;
    do {
      hasMoreRows = scanner.next(results);
      int listSize = results.size();
      for (int i = 0; i < listSize; i++) {
        temp = ci.getValue(colFamily, qualifier, results.get(i));
        if (temp != null) {
          sumVal = ci.add(sumVal, ci.castToReturnType(temp));
        }
      }
      results.clear();
    } while (hasMoreRows);
    if (sumVal != null) {
      response = AggregateResponse.newBuilder().addFirstPart(
        ci.getProtoForPromotedType(sumVal).toByteString()).build();
    }
  } catch (IOException e) {
    CoprocessorRpcUtils.setControllerException(controller, e);
  } finally {
    if (scanner != null) {
      try {
        scanner.close();
      } catch (IOException ignored) {}
    }
  }
  log.debug("Sum from this region is "
      + env.getRegion().getRegionInfo().getRegionNameAsString() + ": " + sum);
  done.run(response);
}
 

/**
 * Gives the row count for the given column family and column qualifier, in
 * the given row range as defined in the Scan object.
 */
@Override
public void getRowNum(RpcController controller, AggregateRequest request,
        RpcCallback<AggregateResponse> done) {
  AggregateResponse response = null;
  long counter = 0L;
  List<Cell> results = new ArrayList<>();
  InternalScanner scanner = null;
  try {
    Scan scan = ProtobufUtil.toScan(request.getScan());
    byte[][] colFamilies = scan.getFamilies();
    byte[] colFamily = colFamilies != null ? colFamilies[0] : null;
    NavigableSet<byte[]> qualifiers = colFamilies != null ?
        scan.getFamilyMap().get(colFamily) : null;
    byte[] qualifier = null;
    if (qualifiers != null && !qualifiers.isEmpty()) {
      qualifier = qualifiers.pollFirst();
    }
    if (scan.getFilter() == null && qualifier == null) {
      scan.setFilter(new FirstKeyOnlyFilter());
    }
    scanner = env.getRegion().getScanner(scan);
    boolean hasMoreRows = false;
    do {
      hasMoreRows = scanner.next(results);
      if (results.size() > 0) {
        counter++;
      }
      results.clear();
    } while (hasMoreRows);
    ByteBuffer bb = ByteBuffer.allocate(8).putLong(counter);
    bb.rewind();
    response = AggregateResponse.newBuilder().addFirstPart(
        ByteString.copyFrom(bb)).build();
  } catch (IOException e) {
    CoprocessorRpcUtils.setControllerException(controller, e);
  } finally {
    if (scanner != null) {
      try {
        scanner.close();
      } catch (IOException ignored) {}
    }
  }
  log.info("Row counter from this region is "
      + env.getRegion().getRegionInfo().getRegionNameAsString() + ": " + counter);
  done.run(response);
}
 

/**
 * Gives a Pair with first object as Sum and second object as row count,
 * computed for a given combination of column qualifier and column family in
 * the given row range as defined in the Scan object. In its current
 * implementation, it takes one column family and one column qualifier (if
 * provided). In case of null column qualifier, an aggregate sum over all the
 * entire column family will be returned.
 * <p>
 * The average is computed in
 * AggregationClient#avg(byte[], ColumnInterpreter, Scan) by
 * processing results from all regions, so its "ok" to pass sum and a Long
 * type.
 */
@Override
public void getAvg(RpcController controller, AggregateRequest request,
        RpcCallback<AggregateResponse> done) {
  AggregateResponse response = null;
  InternalScanner scanner = null;
  try {
    ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request);
    S sumVal = null;
    Long rowCountVal = 0L;
    Scan scan = ProtobufUtil.toScan(request.getScan());
    scanner = env.getRegion().getScanner(scan);
    byte[] colFamily = scan.getFamilies()[0];
    NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
    byte[] qualifier = null;
    if (qualifiers != null && !qualifiers.isEmpty()) {
      qualifier = qualifiers.pollFirst();
    }
    List<Cell> results = new ArrayList<>();
    boolean hasMoreRows = false;

    do {
      results.clear();
      hasMoreRows = scanner.next(results);
      int listSize = results.size();
      for (int i = 0; i < listSize; i++) {
        sumVal = ci.add(sumVal, ci.castToReturnType(ci.getValue(colFamily,
            qualifier, results.get(i))));
      }
      rowCountVal++;
    } while (hasMoreRows);
    if (sumVal != null) {
      ByteString first = ci.getProtoForPromotedType(sumVal).toByteString();
      AggregateResponse.Builder pair = AggregateResponse.newBuilder();
      pair.addFirstPart(first);
      ByteBuffer bb = ByteBuffer.allocate(8).putLong(rowCountVal);
      bb.rewind();
      pair.setSecondPart(ByteString.copyFrom(bb));
      response = pair.build();
    }
  } catch (IOException e) {
    CoprocessorRpcUtils.setControllerException(controller, e);
  } finally {
    if (scanner != null) {
      try {
        scanner.close();
      } catch (IOException ignored) {}
    }
  }
  done.run(response);
}
 

/**
 * Gives a Pair with first object a List containing Sum and sum of squares,
 * and the second object as row count. It is computed for a given combination of
 * column qualifier and column family in the given row range as defined in the
 * Scan object. In its current implementation, it takes one column family and
 * one column qualifier (if provided). The idea is get the value of variance first:
 * the average of the squares less the square of the average a standard
 * deviation is square root of variance.
 */
@Override
public void getStd(RpcController controller, AggregateRequest request,
        RpcCallback<AggregateResponse> done) {
  InternalScanner scanner = null;
  AggregateResponse response = null;
  try {
    ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request);
    S sumVal = null, sumSqVal = null, tempVal = null;
    long rowCountVal = 0L;
    Scan scan = ProtobufUtil.toScan(request.getScan());
    scanner = env.getRegion().getScanner(scan);
    byte[] colFamily = scan.getFamilies()[0];
    NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
    byte[] qualifier = null;
    if (qualifiers != null && !qualifiers.isEmpty()) {
      qualifier = qualifiers.pollFirst();
    }
    List<Cell> results = new ArrayList<>();

    boolean hasMoreRows = false;

    do {
      tempVal = null;
      hasMoreRows = scanner.next(results);
      int listSize = results.size();
      for (int i = 0; i < listSize; i++) {
        tempVal = ci.add(tempVal, ci.castToReturnType(ci.getValue(colFamily,
            qualifier, results.get(i))));
      }
      results.clear();
      sumVal = ci.add(sumVal, tempVal);
      sumSqVal = ci.add(sumSqVal, ci.multiply(tempVal, tempVal));
      rowCountVal++;
    } while (hasMoreRows);
    if (sumVal != null) {
      ByteString first_sumVal = ci.getProtoForPromotedType(sumVal).toByteString();
      ByteString first_sumSqVal = ci.getProtoForPromotedType(sumSqVal).toByteString();
      AggregateResponse.Builder pair = AggregateResponse.newBuilder();
      pair.addFirstPart(first_sumVal);
      pair.addFirstPart(first_sumSqVal);
      ByteBuffer bb = ByteBuffer.allocate(8).putLong(rowCountVal);
      bb.rewind();
      pair.setSecondPart(ByteString.copyFrom(bb));
      response = pair.build();
    }
  } catch (IOException e) {
    CoprocessorRpcUtils.setControllerException(controller, e);
  } finally {
    if (scanner != null) {
      try {
        scanner.close();
      } catch (IOException ignored) {}
    }
  }
  done.run(response);
}
 

/**
 * Gives a List containing sum of values and sum of weights.
 * It is computed for the combination of column
 * family and column qualifier(s) in the given row range as defined in the
 * Scan object. In its current implementation, it takes one column family and
 * two column qualifiers. The first qualifier is for values column and
 * the second qualifier (optional) is for weight column.
 */
@Override
public void getMedian(RpcController controller, AggregateRequest request,
        RpcCallback<AggregateResponse> done) {
  AggregateResponse response = null;
  InternalScanner scanner = null;
  try {
    ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request);
    S sumVal = null, sumWeights = null, tempVal = null, tempWeight = null;
    Scan scan = ProtobufUtil.toScan(request.getScan());
    scanner = env.getRegion().getScanner(scan);
    byte[] colFamily = scan.getFamilies()[0];
    NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily);
    byte[] valQualifier = null, weightQualifier = null;
    if (qualifiers != null && !qualifiers.isEmpty()) {
      valQualifier = qualifiers.pollFirst();
      // if weighted median is requested, get qualifier for the weight column
      weightQualifier = qualifiers.pollLast();
    }
    List<Cell> results = new ArrayList<>();

    boolean hasMoreRows = false;

    do {
      tempVal = null;
      tempWeight = null;
      hasMoreRows = scanner.next(results);
      int listSize = results.size();
      for (int i = 0; i < listSize; i++) {
        Cell kv = results.get(i);
        tempVal = ci.add(tempVal, ci.castToReturnType(ci.getValue(colFamily,
            valQualifier, kv)));
        if (weightQualifier != null) {
          tempWeight = ci.add(tempWeight,
              ci.castToReturnType(ci.getValue(colFamily, weightQualifier, kv)));
        }
      }
      results.clear();
      sumVal = ci.add(sumVal, tempVal);
      sumWeights = ci.add(sumWeights, tempWeight);
    } while (hasMoreRows);
    ByteString first_sumVal = ci.getProtoForPromotedType(sumVal).toByteString();
    S s = sumWeights == null ? ci.castToReturnType(ci.getMinValue()) : sumWeights;
    ByteString first_sumWeights = ci.getProtoForPromotedType(s).toByteString();
    AggregateResponse.Builder pair = AggregateResponse.newBuilder();
    pair.addFirstPart(first_sumVal);
    pair.addFirstPart(first_sumWeights);
    response = pair.build();
  } catch (IOException e) {
    CoprocessorRpcUtils.setControllerException(controller, e);
  } finally {
    if (scanner != null) {
      try {
        scanner.close();
      } catch (IOException ignored) {}
    }
  }
  done.run(response);
}
 

@Test
public void testScan() throws Exception {

  byte[] startRow = Bytes.toBytes("startRow");
  byte[] stopRow = Bytes.toBytes("stopRow");
  byte[] fam = Bytes.toBytes("fam");
  byte[] qf1 = Bytes.toBytes("qf1");

  long ts = System.currentTimeMillis();
  int maxVersions = 2;

  Scan scan = new Scan().withStartRow(startRow).withStopRow(stopRow);
  scan.addColumn(fam, qf1);
  scan.setTimeRange(ts, ts + 1);
  scan.readVersions(maxVersions);

  ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan);
  Scan desScan = ProtobufUtil.toScan(scanProto);

  assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
  assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
  assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
  Set<byte[]> set = null;
  Set<byte[]> desSet = null;

  for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
    assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
    set = entry.getValue();
    desSet = desScan.getFamilyMap().get(entry.getKey());
    for (byte[] column : set) {
      assertTrue(desSet.contains(column));
    }

    // Test filters are serialized properly.
    scan = new Scan().withStartRow(startRow);
    final String name = "testScan";
    byte[] prefix = Bytes.toBytes(name);
    scan.setFilter(new PrefixFilter(prefix));
    scanProto = ProtobufUtil.toScan(scan);
    desScan = ProtobufUtil.toScan(scanProto);
    Filter f = desScan.getFilter();
    assertTrue(f instanceof PrefixFilter);
  }

  assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
  TimeRange tr = scan.getTimeRange();
  TimeRange desTr = desScan.getTimeRange();
  assertEquals(tr.getMax(), desTr.getMax());
  assertEquals(tr.getMin(), desTr.getMin());
}
 

@Override
protected ResultScanner getScanner(Scan scan) throws IOException{
    if (!useProxy) {
        try {
            return delegate.getScanner(scan);
        } catch (AccessDeniedException ade) {
            activateProxy(ade);
        }
    }

    try {
        try (java.sql.Connection jdbcConnection = connectionPool.getConnection();
             PreparedStatement statement = jdbcConnection.prepareStatement("call SYSCS_UTIL.SYSCS_HBASE_OPERATION(?, ?, ?)")) {
            statement.setString(1, tableName.toString());
            statement.setString(2, "scan");
            Scan copy = new Scan(scan);
            copy.setSmall(true);
            ClientProtos.Scan scanRequest = ProtobufUtil.toScan(copy);
            statement.setBlob(3, new ArrayInputStream(scanRequest.toByteArray()));
            try (ResultSet rs = statement.executeQuery()) {
                Queue<Result> results = new ArrayDeque<>();
                while (rs.next()) {
                    Blob blob = rs.getBlob(1);
                    byte[] bytes = blob.getBytes(1, (int) blob.length());

                    ClientProtos.Result result = ClientProtos.Result.parseFrom(bytes);
                    results.add(ProtobufUtil.toResult(result));
                }
                return new ResultScanner() {
                    @Override
                    public Result next() throws IOException {
                        return results.poll();
                    }

                    @Override
                    public Result[] next(int nbRows) throws IOException {
                        int size = Math.min(nbRows, results.size());
                        List<Result> r = new ArrayList<>(size);
                        while(size-- > 0) {
                            r.add(results.poll());
                        }
                        return r.toArray(new Result[size]);
                    }

                    @Override
                    public void close() {
                        // nothing
                    }

                    public boolean renewLease() {
                        return false;
                    }

                    public ScanMetrics getScanMetrics() {
                        return null;
                    }

                    @Override
                    public Iterator<Result> iterator() {
                        return results.iterator();
                    }
                };
            }
        }
    } catch (SQLException e) {
        throw new IOException(e);
    }
}
 

/**
 * Writes the given scan into a Base64 encoded string.
 *
 * @param scan  The scan to write out.
 * @return The scan saved in a Base64 encoded string.
 * @throws IOException When writing the scan fails.
 */
public static String convertScanToString(Scan scan) throws IOException {
  ClientProtos.Scan proto = ProtobufUtil.toScan(scan);
  return Bytes.toString(Base64.getEncoder().encode(proto.toByteArray()));
}
 

/**
 * Converts the given Base64 string back into a Scan instance.
 *
 * @param base64  The scan details.
 * @return The newly created Scan instance.
 * @throws IOException When reading the scan instance fails.
 */
public static Scan convertStringToScan(String base64) throws IOException {
  byte [] decoded = Base64.getDecoder().decode(base64);
  return ProtobufUtil.toScan(ClientProtos.Scan.parseFrom(decoded));
}