org.apache.kudu.client.AsyncKuduScanner Java Examples

/**
 * Retrieves the tagset IDs of all tagsets which contain the specified tag.
 * The tagset IDs are returned in sorted order.
 *
 * @param key the tag key
 * @param value the tag value
 * @return the sorted tagset IDs
 */
public Deferred<IntVec> getTagsetIDsForTag(final String key, final String value) {
  AsyncKuduScanner.AsyncKuduScannerBuilder scan = client.newScannerBuilder(table);
  scan.addPredicate(KuduPredicate.newComparisonPredicate(Tables.TAGS_KEY_COLUMN,
                                                         ComparisonOp.EQUAL, key));
  scan.addPredicate(KuduPredicate.newComparisonPredicate(Tables.TAGS_VALUE_COLUMN,
                                                         ComparisonOp.EQUAL, value));
  scan.setProjectedColumnIndexes(TAGSET_ID_PROJECTION);
  final AsyncKuduScanner scanner = scan.build();

  class GetTagCB implements Callback<Deferred<IntVec>, RowResultIterator> {
    private final IntVec tagsetIDs = IntVec.create();
    @Override
    public Deferred<IntVec> call(RowResultIterator results) {
      for (RowResult result : results) {
        tagsetIDs.push(result.getInt(0));
      }
      if (scanner.hasMoreRows()) {
        return scanner.nextRows().addCallbackDeferring(this);
      }
      // The Kudu java client doesn't yet allow us to specify a sorted
      // (fault-tolerant) scan, so have to sort manually.
      tagsetIDs.sort();
      return Deferred.fromResult(tagsetIDs);
    }
    @Override
    public String toString() {
      return MoreObjects.toStringHelper(this).add("key", key).add("value", value).toString();
    }
  }

  return scanner.nextRows().addCallbackDeferring(new GetTagCB());
}
 

private Deferred<TagsetLookupResult> lookupTagset(SerializedTagset tagset, int id) {
  LOG.debug("Looking up tagset; id: {}, tags: {}", id, tagset);
  AsyncKuduScanner tagsetScanner = tagsetScanner(id);

  return tagsetScanner.nextRows().addCallbackDeferring(
      new TagsetScanCB(tagset, id, tagsetScanner));
}
 

/**
 * Creates an {@link AsyncKuduScanner} over the tagset table beginning with
 * the specified ID.
 *
 * @param id the ID to begin scanning from
 * @return the scanner
 */
private AsyncKuduScanner tagsetScanner(int id) {
  AsyncKuduScanner.AsyncKuduScannerBuilder scanBuilder = client.newScannerBuilder(tagsetsTable);
  scanBuilder.addPredicate(KuduPredicate.newComparisonPredicate(Tables.TAGSETS_ID_COLUMN,
                                                                ComparisonOp.GREATER_EQUAL,
                                                                id));
  if (id < Integer.MAX_VALUE - TAGSETS_PER_SCAN) {
    scanBuilder.addPredicate(KuduPredicate.newComparisonPredicate(Tables.TAGSETS_ID_COLUMN,
                                                                  ComparisonOp.LESS,
                                                                  id + TAGSETS_PER_SCAN));
  }
  scanBuilder.setProjectedColumnIndexes(columnIndexes);
  AbstractionBulldozer.sortResultsByPrimaryKey(scanBuilder);
  return scanBuilder.build();
}
 

public void checkFinalize(
    final AsyncKuduScanner scanner,
    final Semaphore semaphore,
    final BlockingQueue<Object> resultQueue,
    final AtomicInteger queryCount) {
  scanner.close();
  semaphore.release();
  if (queryCount.decrementAndGet() <= 0) {
    try {
      resultQueue.put(RowConsumer.POISON);
    } catch (final InterruptedException e) {
      LOGGER.error("Interrupted while finishing blocking queue, this may result in deadlock!");
    }
  }
}
 

/***
 * The main logic which takes the parsed in query and builds the Kudud scan tokens specific to this query.
 * It makes sure that these scan tokens are sorted before the actual scan tokens that are to be executed in the
 * current physical instance of the operator are shortlisted. Since the kudu scan taken builder gives the scan
 * tokens for the query and does not differentiate between a distributed system and a single instance system, this
 * method takes the plan as generated by the Kudu scan token builder and then chooses only those segments that were
 * decided to be the responsibility of this operator at partitioning time.
 * @param parsedQuery The parsed query instance
 * @return A list of partition scan metadata objects that are applicable for this instance of the physical operator
 * i.e. the operator owning this instance of the scanner.
 * @throws IOException If the scan assignment cannot be serialized
 */
public List<KuduPartitionScanAssignmentMeta> preparePlanForScanners(SQLToKuduPredicatesTranslator parsedQuery)
  throws IOException
{
  List<KuduPredicate> predicateList = parsedQuery.getKuduSQLParseTreeListener().getKuduPredicateList();
  ApexKuduConnection apexKuduConnection = verifyConnectionStaleness(0);// we will have atleast one connection
  KuduScanToken.KuduScanTokenBuilder builder = apexKuduConnection.getKuduClient().newScanTokenBuilder(
      apexKuduConnection.getKuduTable());
  builder = builder.setProjectedColumnNames(new ArrayList<>(
      parsedQuery.getKuduSQLParseTreeListener().getListOfColumnsUsed()));
  for (KuduPredicate aPredicate : predicateList) {
    builder = builder.addPredicate(aPredicate);
  }
  builder.setFaultTolerant(parentOperator.isFaultTolerantScanner());
  Map<String,String> optionsUsedForThisQuery = parentOperator.getOptionsEnabledForCurrentQuery();
  if ( optionsUsedForThisQuery.containsKey(KuduSQLParseTreeListener.READ_SNAPSHOT_TIME)) {
    try {
      long readSnapShotTime = Long.valueOf(optionsUsedForThisQuery.get(KuduSQLParseTreeListener.READ_SNAPSHOT_TIME));
      builder = builder.readMode(AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT);
      builder = builder.snapshotTimestampMicros(readSnapShotTime);
      LOG.info("Using read snapshot for this query as " + readSnapShotTime);
    } catch ( Exception ex) {
      LOG.error("Cannot parse the Read snaptshot time " + ex.getMessage(), ex);
    }
  }
  List<KuduScanToken> allPossibleScanTokens = builder.build();
  Collections.sort(allPossibleScanTokens, // Make sure we deal with a sorted list of scan tokens
      new Comparator<KuduScanToken>()
    {
      @Override
      public int compare(KuduScanToken left, KuduScanToken right)
      {
        return left.compareTo(right);
      }
    });
  LOG.info(" Query will scan " + allPossibleScanTokens.size() + " tablets");
  if ( LOG.isDebugEnabled()) {
    LOG.debug(" Predicates scheduled for this query are " + predicateList.size());
    for ( int i = 0; i < allPossibleScanTokens.size(); i++) {
      LOG.debug("A tablet scheduled for all operators scanning is " + allPossibleScanTokens.get(i).getTablet());
    }
  }
  List<KuduPartitionScanAssignmentMeta> partitionPieForThisOperator = parentOperator.getPartitionPieAssignment();
  List<KuduPartitionScanAssignmentMeta> returnOfAssignments = new ArrayList<>();
  int totalScansForThisQuery = allPossibleScanTokens.size();
  int counterForPartAssignments = 0;
  for (KuduPartitionScanAssignmentMeta aPartofThePie : partitionPieForThisOperator) {
    if ( aPartofThePie.getOrdinal() < totalScansForThisQuery) { // a given query plan might have less scantokens
      KuduPartitionScanAssignmentMeta aMetaForThisQuery = new KuduPartitionScanAssignmentMeta();
      aMetaForThisQuery.setTotalSize(totalScansForThisQuery);
      aMetaForThisQuery.setOrdinal(counterForPartAssignments);
      counterForPartAssignments += 1;
      aMetaForThisQuery.setCurrentQuery(parsedQuery.getSqlExpresssion());
      // we pick up only those ordinals that are part of the original partition pie assignment
      KuduScanToken aTokenForThisOperator = allPossibleScanTokens.get(aPartofThePie.getOrdinal());
      aMetaForThisQuery.setSerializedKuduScanToken(aTokenForThisOperator.serialize());
      returnOfAssignments.add(aMetaForThisQuery);
      LOG.debug("Added query scan for this operator " + aMetaForThisQuery + " with scan tablet as " +
          allPossibleScanTokens.get(aPartofThePie.getOrdinal()).getTablet());
    }
  }
  LOG.info(" A total of " + returnOfAssignments.size() + " have been scheduled for this operator");
  return returnOfAssignments;
}
 

/**
 * Create a new {@code TagsetScanCB} looking for a tagset starting with the provided ID.
 *
 * @param tagset  the tagset being looked up
 * @param id      the ID that the scanner is looking up
 * @param scanner the initialscanner
 */
TagsetScanCB(SerializedTagset tagset, int id, AsyncKuduScanner scanner) {
  this.tagset = tagset;
  this.scanner = scanner;
  this.id = id;
  this.probe = id;
}