Java Code Examples for org.apache.accumulo.core.data.Key#getRow()

/**
 * Converts from an Accumulo Key to a Fluo RowColumn
 * 
 * @param key Key
 * @return RowColumn
 */
public static RowColumn toRowColumn(Key key) {
  if (key == null) {
    return RowColumn.EMPTY;
  }
  if ((key.getRow() == null) || key.getRow().getLength() == 0) {
    return RowColumn.EMPTY;
  }
  Bytes row = ByteUtil.toBytes(key.getRow());
  if ((key.getColumnFamily() == null) || key.getColumnFamily().getLength() == 0) {
    return new RowColumn(row);
  }
  Bytes cf = ByteUtil.toBytes(key.getColumnFamily());
  if ((key.getColumnQualifier() == null) || key.getColumnQualifier().getLength() == 0) {
    return new RowColumn(row, new Column(cf));
  }
  Bytes cq = ByteUtil.toBytes(key.getColumnQualifier());
  if ((key.getColumnVisibility() == null) || key.getColumnVisibility().getLength() == 0) {
    return new RowColumn(row, new Column(cf, cq));
  }
  Bytes cv = ByteUtil.toBytes(key.getColumnVisibility());
  return new RowColumn(row, new Column(cf, cq, cv));
}
 

protected Key getEndKey(JexlNode node) {
    Key endKey = rangeLimiter.getEndKey();
    String identifier = JexlASTHelper.getIdentifier(node);
    Object objValue = JexlASTHelper.getLiteralValue(node);
    String value = null == objValue ? "null" : objValue.toString();
    
    StringBuilder builder = new StringBuilder("fi");
    builder.append(NULL_DELIMETER).append(identifier);
    
    Text cf = new Text(builder.toString());
    
    builder = new StringBuilder(value);
    
    builder.append(NULL_DELIMETER).append(endKey.getColumnFamily());
    Text cq = new Text(builder.toString());
    
    return new Key(endKey.getRow(), cf, cq, endKey.getTimestamp());
}
 

/**
 * Advance an iterator until skip(...) returns false. May be a combination of seek() and next() calls
 * 
 * @param itr
 * @param pointer
 * @param currentRange
 * @param columnFamilies
 * @param includeColumnFamilies
 * @throws IOException
 */
protected void advanceItr(SortedKeyValueIterator<Key,Value> itr, ByteSequence pointer, Range currentRange, Collection<ByteSequence> columnFamilies,
                boolean includeColumnFamilies) throws IOException {
    Key current = itr.getTopKey();
    Text row = current.getRow();
    int nextCount = 0;
    while (current != null && skip(current, row, pointer)) {
        if (maxNextCount == -1 || nextCount < maxNextCount) {
            itr.next();
            nextCount++;
        } else {
            Key startKey = getSeekStartKey(current, pointer);
            Range newRange = new Range(startKey, false, currentRange.getEndKey(), currentRange.isEndKeyInclusive());
            itr.seek(newRange, columnFamilies, includeColumnFamilies);
            nextCount = 0;
        }
        
        current = itr.hasTop() ? itr.getTopKey() : null;
    }
}
 

private Key newDocumentKey(final Key fieldKey, long timestamp) {
    final Text row = fieldKey.getRow();
    final Text cf = fieldKey.getColumnFamily();
    final Text cq = new Text();
    final Text visibility = new Text();
    return new Key(row, cf, cq, visibility, timestamp);
}
 

/**
 * Computes a "stats" metric value. Each incoming value in {@code values} is assumed to be a unique value. We calculate min, max, median, and average values
 * and include those in the output mutation. Note that median will only be calculated if there are not more than {@code MAX_MEDIAN_COUNT} values. This
 * restriction prevents us from using too much memory in the task tracker.
 */
private Mutation statsMutation(Key key, Iterable<Value> values) {
    long numLongs = 0;
    long min = Long.MAX_VALUE;
    long max = Long.MIN_VALUE;
    longs.clear();
    LongValueSum sum = new LongValueSum();
    for (Value v : values) {
        ++numLongs;
        long aLong = Long.parseLong(v.toString());
        min = Math.min(aLong, min);
        max = Math.max(aLong, max);
        sum.addNextValue(aLong);
        
        if (numLongs <= MAX_MEDIAN_COUNT)
            longs.add(aLong);
    }
    numLongs = Math.max(1, numLongs);
    
    String average = String.format("%1.4f", sum.getSum() / (double) numLongs);
    
    ColumnVisibility columnVisibility = new ColumnVisibility(key.getColumnVisibility());
    Text columnFamily = key.getColumnFamily();
    Mutation m = new Mutation(key.getRow());
    m.put(columnFamily, MIN_TEXT, columnVisibility, new Value(Long.toString(min).getBytes()));
    m.put(columnFamily, MAX_TEXT, columnVisibility, new Value(Long.toString(max).getBytes()));
    m.put(columnFamily, AVERAGE_TEXT, columnVisibility, new Value(average.getBytes()));
    if (numLongs <= MAX_MEDIAN_COUNT) {
        Collections.sort(longs);
        String median = "" + longs.get(longs.size() / 2);
        m.put(columnFamily, MEDIAN_TEXT, columnVisibility, new Value(median.getBytes()));
    }
    return m;
}
 

/**
 * Advance the source to the Key specified by pointer then fetch the next tk/tv from that point
 * 
 * @param pointer
 *            the minimum point to advance source to
 * @throws IOException
 * @throws IllegalStateException
 *             if getTopKey() is greater than or equal to pointer
 */
@Override
public void move(Key pointer) throws IOException {
    if (this.hasTop() && this.getTopKey().compareTo(pointer) >= 0) {
        throw new IllegalStateException("Tried to called move when we were already at or beyond where we were told to move to: topkey=" + this.getTopKey()
                        + ", movekey=" + pointer);
    }
    
    newColumnQualifier.set(valueMinPrefix);
    Text id = pointer.getColumnFamily();
    newColumnQualifier.append(id.getBytes(), 0, id.getLength());
    
    Key nextKey = new Key(pointer.getRow(), columnFamily, newColumnQualifier);
    Key newTop = null;
    for (int i = 0; i < 256 && source.hasTop() && (newTop = source.getTopKey()).compareTo(nextKey) < 0; ++i)
        source.next();
    
    /*
     * We need to verify a few things after next()'ing a bunch and then seeking:
     * 
     * 1) We actually had data before even trying to move 2) The last returned key by the source is less than the one we want to be at 3) The source still
     * has data - we could get away without checking this, but why try and seek if we already know we have no more data?
     */
    if (newTop != null && newTop.compareTo(nextKey) < 0 && source.hasTop()) {
        Range r = new Range(nextKey, true, scanRange.getEndKey(), scanRange.isEndKeyInclusive());
        if (log.isTraceEnabled())
            log.trace(this + " move'ing to: " + r);
        source.seek(r, seekColumnFamilies, includeColumnFamilies);
    } else {
        if (log.isTraceEnabled())
            log.trace(this + " stepping its way to " + newTop);
    }
    
    if (log.isTraceEnabled()) {
        log.trace(this + " finished move. Now at " + (source.hasTop() ? source.getTopKey() : "null") + ", calling next()");
    }
    
    next();
}
 

private Key newResultKey(final Entry<Key,Document> from) {
    final Key key = from.getKey();
    final Text row = key.getRow();
    final Text cf = key.getColumnFamily();
    final Text cq = key.getColumnQualifier();
    final Text visibility = key.getColumnVisibility();
    long timestamp = from.getValue().getTimestamp();
    return new Key(row, cf, cq, visibility, timestamp);
}
 

/**
 * Don't accept this key if the colf starts with 'fi'
 */
@Override
public boolean isKeyAccepted(Key key) throws IOException {
    if (key.getColumnFamily().toString().startsWith("fi")) {
        Key copy = new Key(key.getRow(), new Text("fi\01"));
        Collection<ByteSequence> columnFamilies = Collections.emptyList();
        this.iterator.seek(new Range(copy, copy), columnFamilies, true);
        if (this.iterator.hasTop())
            return isKeyAccepted(this.iterator.getTopKey());
        return true;
    }
    return true;
}
 

public Key apply(SortedKeyValueIterator<Key,Value> itr, Document d, AttributeFactory af) throws IOException {
    Key key = itr.getTopKey();
    ByteSequence parentId = parseRootPointerFromFI(key.getColumnQualifierData());
    Text row = key.getRow();
    ByteSequence docId = null;
    Key nextKey = key;
    do {
        key = nextKey;
        String field = key.getColumnFamily().toString().substring(3);
        String value = key.getColumnQualifier().toString();
        value = value.substring(0, value.indexOf('\0'));
        Attribute<?> attr = af.create(field, value, key, true);
        // in addition to keeping fields that the filter indicates should be kept, also keep fields that the filter applies. This is due to inconsistent
        // behavior between event/tld queries where an index only field index will be kept except when it is a child of a tld
        attr.setToKeep((fieldsToAggregate == null || fieldsToAggregate.contains(JexlASTHelper.removeGroupingContext(field)))
                        && (attrFilter == null || attrFilter.keep(key)));
        d.put(field, attr);
        
        ByteSequence thisId = parsePointerFromFI(key.getColumnQualifierData());
        if (docId == null || !docId.equals(thisId)) {
            docId = thisId;
            Key docKey = new Key(key.getRow(), new Text(docId.toArray()), new Text(), ColumnVisibilityCache.get(key.getColumnVisibilityData()),
                            key.getTimestamp());
            attr = new DocumentKey(docKey, false);
            d.put(Document.DOCKEY_FIELD_NAME, attr);
        }
        itr.next();
        nextKey = itr.hasTop() ? itr.getTopKey() : null;
    } while (skip(nextKey, row, parentId));
    return getResult(key, parentId);
}
 

protected Text getRow(Key key) {
  return key.getRow();
}
 

private boolean jumpSeek(Range r) throws IOException {
    range = r;
    setTopKey(null);
    setTopValue(null);
    getSource().seek(range, EMPTY_COL_FAMS, false);
    while (topKey == null) {
        if (getSource().hasTop()) {
            if (log.isDebugEnabled()) {
                log.debug("jump, source has top: " + getSource().getTopKey());
            }
            Key k = getSource().getTopKey();
            if (range.contains(k)) {
                if (matches(k)) {
                    topKey = k;
                    topValue = getSource().getTopValue();
                    if (log.isDebugEnabled()) {
                        log.debug("jump, source has top in valid range");
                    }
                } else {
                    getSource().next();
                }
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("jump, top out of range");
                    String pEndRow = "empty";
                    if (parentEndRow != null) {
                        pEndRow = parentEndRow.toString();
                    }
                    log.debug("source.topKey.row: " + k.getRow() + "\t currentRow: " + currentRow + "\t parentEndRow: " + pEndRow);
                }

                if (parentEndRow != null) {

                    if (currentRow == null) {
                        topKey = null;
                        topValue = null;
                        return false;
                    }
                    // check it
                    if (k.getRow().equals(currentRow)) {
                        currentRow = getNextRow();
                    } else if (k.getRow().compareTo(currentRow) > 0) {
                        currentRow = k.getRow();
                    }

                    if (currentRow == null || parentEndRow.compareTo(currentRow) < 0) {
                        // you're done
                        topKey = null;
                        topValue = null;
                        return false;
                    }

                } else { // can go to end of the tablet
                    if (currentRow == null || k.getRow() == null) {
                        topKey = null;
                        topValue = null;
                        return false;
                    }

                    if (k.getRow().equals(currentRow)) {
                        currentRow = getNextRow();
                        if (currentRow == null) {
                            topKey = null;
                            topValue = null;
                            return false;
                        }
                    } else if (k.getRow().compareTo(currentRow) > 0) {
                        currentRow = k.getRow();
                    }
                }

                // construct new range and seek the source
                range = buildRange(currentRow);
                if (log.isDebugEnabled()) {
                    log.debug("jump, new build range: " + range);
                }
                getSource().seek(range, EMPTY_COL_FAMS, false);
            }
        } else {
            if (log.isDebugEnabled()) {
                log.debug("jump, underlying source had no top key.");
            }
            topKey = null;
            topValue = null;
            return false;
        }
    }// end while

    return hasTop();
}
 

/**
 * Gets the TabletServer hostname for where the given key is located in the given table
 *
 * @param table Fully-qualified table name
 * @param key Key to locate
 * @return The tablet location, or DUMMY_LOCATION if an error occurs
 */
private Optional<String> getTabletLocation(String table, Key key)
{
    try {
        // Get the Accumulo table ID so we can scan some fun stuff
        String tableId = connector.tableOperations().tableIdMap().get(table);

        // Create our scanner against the metadata table, fetching 'loc' family
        Scanner scanner = connector.createScanner("accumulo.metadata", auths);
        scanner.fetchColumnFamily(new Text("loc"));

        // Set the scan range to just this table, from the table ID to the default tablet
        // row, which is the last listed tablet
        Key defaultTabletRow = new Key(tableId + '<');
        Key start = new Key(tableId);
        Key end = defaultTabletRow.followingKey(PartialKey.ROW);
        scanner.setRange(new Range(start, end));

        Optional<String> location = Optional.empty();
        if (key == null) {
            // if the key is null, then it is -inf, so get first tablet location
            Iterator<Entry<Key, Value>> iter = scanner.iterator();
            if (iter.hasNext()) {
                location = Optional.of(iter.next().getValue().toString());
            }
        }
        else {
            // Else, we will need to scan through the tablet location data and find the location

            // Create some text objects to do comparison for what we are looking for
            Text splitCompareKey = new Text();
            key.getRow(splitCompareKey);
            Text scannedCompareKey = new Text();

            // Scan the table!
            for (Entry<Key, Value> entry : scanner) {
                // Get the bytes of the key
                byte[] keyBytes = entry.getKey().getRow().copyBytes();

                // If the last byte is <, then we have hit the default tablet, so use this location
                if (keyBytes[keyBytes.length - 1] == '<') {
                    location = Optional.of(entry.getValue().toString());
                    break;
                }
                else {
                    // Chop off some magic nonsense
                    scannedCompareKey.set(keyBytes, 3, keyBytes.length - 3);

                    // Compare the keys, moving along the tablets until the location is found
                    if (scannedCompareKey.getLength() > 0) {
                        int compareTo = splitCompareKey.compareTo(scannedCompareKey);
                        if (compareTo <= 0) {
                            location = Optional.of(entry.getValue().toString());
                        }
                        else {
                            // all future tablets will be greater than this key
                            break;
                        }
                    }
                }
            }
            scanner.close();
        }

        // If we were unable to find the location for some reason, return the default tablet
        // location
        return location.isPresent() ? location : getDefaultTabletLocation(table);
    }
    catch (Exception e) {
        // Swallow this exception so the query does not fail due to being unable
        // to locate the tablet server for the provided Key.
        // This is purely an optimization, but we will want to log the error.
        LOG.error("Failed to get tablet location, returning dummy location", e);
        return Optional.empty();
    }
}
 

@Override
protected Key getSeekStartKey(Key current, ByteSequence pointer) {
    // CQ = dataType\0UID\0Normalized field value\0Field name
    // seek to the next documents TF
    return new Key(current.getRow(), current.getColumnFamily(), new Text(pointer + Constants.NULL_BYTE_STRING + Constants.MAX_UNICODE_STRING));
}
 

protected void run(String[] args) throws ParseException, AccumuloSecurityException, AccumuloException, TableNotFoundException, IOException {
    parseConfig(args);
    
    ZooKeeperInstance instance = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instanceName).withZkHosts(zookeepers));
    Connector connector = instance.getConnector(username, new PasswordToken(password));
    Authorizations auths = connector.securityOperations().getUserAuthorizations(connector.whoami());
    
    try (BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig().setMaxWriteThreads(bwThreads).setMaxMemory(bwMemory)
                    .setMaxLatency(60, TimeUnit.SECONDS));
                    BatchScanner scanner = connector.createBatchScanner(tableName, auths, bsThreads)) {
        scanner.setRanges(ranges);
        
        for (Entry<Key,Value> entry : scanner) {
            Key key = entry.getKey();
            
            ByteArrayDataInput in = ByteStreams.newDataInput(entry.getValue().get());
            Counters counters = new Counters();
            try {
                counters.readFields(in);
            } catch (IOException e) {
                // The IO exception means the counters are in the wrong format. We *assume* that they are in
                // the old (CDH3) format, and de-serialize according to that, and re-write the key with the new value.
                in = ByteStreams.newDataInput(entry.getValue().get());
                int numGroups = in.readInt();
                while (numGroups-- > 0) {
                    String groupName = Text.readString(in);
                    String groupDisplayName = Text.readString(in);
                    CounterGroup group = counters.addGroup(groupName, groupDisplayName);
                    
                    int groupSize = WritableUtils.readVInt(in);
                    for (int i = 0; i < groupSize; i++) {
                        String counterName = Text.readString(in);
                        String counterDisplayName = counterName;
                        if (in.readBoolean())
                            counterDisplayName = Text.readString(in);
                        long value = WritableUtils.readVLong(in);
                        group.addCounter(counterName, counterDisplayName, value);
                    }
                }
                
                ByteArrayDataOutput out = ByteStreams.newDataOutput();
                counters.write(out);
                Mutation m = new Mutation(key.getRow());
                m.put(key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibilityParsed(), key.getTimestamp() + 1,
                                new Value(out.toByteArray()));
                writer.addMutation(m);
            }
        }
        
    }
}
 

@Override
public int getPartition(Key key, Value value, int numPartitions) {
    key.getRow(row);
    return (row.hashCode() >>> 1) % numPartitions;
}
 

public void next() throws IOException {
    if (log.isDebugEnabled()) {
        log.debug("next");
    }
    if (key != null) {
        key = null;
        value = null;
    }

    if (eventSpecificRange) {
        // Then this will probably return nothing
        event.next();
        if (event.hasTop()) {
            key = event.getTopKey();
            value = event.getTopValue();
        }
    } else {

        do {
            index.next();
            // If the index has a match, then seek the event to the key
            if (index.hasTop()) {
                Key eventKey = index.getTopKey();
                Key endKey = eventKey.followingKey(PartialKey.ROW_COLFAM);
                Key startKey = new Key(eventKey.getRow(), eventKey.getColumnFamily());
                Range eventRange = new Range(startKey, endKey);
                HashSet<ByteSequence> cf = new HashSet<ByteSequence>();
                cf.add(eventKey.getColumnFamilyData());
                event.seek(eventRange, cf, true);
                if (event.hasTop()) {
                    key = event.getTopKey();
                    value = event.getTopValue();
                }
            }
        } while (key == null && index.hasTop());
    }
    // Sanity check. Make sure both returnValue and returnKey are null or both are not null
    if (!((key == null && value == null) || (key != null && value != null))) {
        log.warn("Key: " + ((key == null) ? "null" : key.toString()));
        log.warn("Value: " + ((value == null) ? "null" : value.toString()));
        throw new IOException("Return values are inconsistent");
    }

}
 

private Key addKeyCount(Key key) {
    resultCount.getAndIncrement();
    return new Key(key.getRow(), new Text(NumericalEncoder.encode(Long.toString(resultCount.get())) + '\0' + key.getColumnFamily()),
                    key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp());
}
 

/**
 * Makes sure only those rows with an index row are kept around. The versioning iterator should have run already
 * and evicted older index rows.
 *
 * @param key
 * @param value
 * @return
 */
@Override
public boolean accept(Key key, Value value) {

    try {
        // first find out if we are inside of an index row
        if (key.getColumnFamily().toString().equals(NULL_BYTE + "INDEX")) {

            if (!key.getRow().toString().equals(currentIndex)) {
                currentIndex = key.getRow().toString();
                uuidSet = new HashSet<String>();
            }

            uuidSet.add(new String(value.get()));

            return true;
        }

        // otherwise, assume we are in an event row
        else {

            String uuid = key.getColumnFamily().toString().replace(END_BYTE, "");
            String hash = new String(value.get());

            if (!uuidSet.contains(uuid + NULL_BYTE + hash)) {
                return false;
            }

            String[] keyValue = key.getColumnQualifier().toString().split(NULL_BYTE);

            // here we want to make sure that any duplicate events added are filtered out (this is possible simply
            // because the maxVersions > 1)

            if (previousEvent != null && previousEvent.equals(key.getRow() + NULL_BYTE + uuid + NULL_BYTE + hash
                    + NULL_BYTE + keyValue[0] + NULL_BYTE + keyValue[1])) {
                return false;
            }

            previousEvent = key.getRow() + NULL_BYTE + uuid + NULL_BYTE + hash + NULL_BYTE
                    + keyValue[0] + NULL_BYTE + keyValue[1];

        }
    } catch (Exception e) {

        return true;
    }

    return true;
}
 

public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
    if (log.isDebugEnabled()) {
        log.debug("seek, range:" + range);
    }
    // Test the range to see if it is event specific.
    if (null != range.getEndKey() && range.getEndKey().getColumnFamily() != null && range.getEndKey().getColumnFamily().getLength() != 0) {
        if (log.isDebugEnabled()) {
            log.debug("Jumping straight to the event");
        }
        // Then this range is for a specific event. We don't need to use the index iterator to find it, we can just
        // seek to it with the event iterator and evaluate it.
        eventSpecificRange = true;
        event.seek(range, columnFamilies, inclusive);
        if (event.hasTop()) {
            key = event.getTopKey();
            value = event.getTopValue();
        }
    } else {
        if (log.isDebugEnabled()) {
            log.debug("Using BooleanLogicIteratorJexl");
        }
        // Seek the boolean logic iterator
        index.seek(range, columnFamilies, inclusive);

        // If the index has a match, then seek the event to the key
        if (index.hasTop()) {
            Key eventKey = index.getTopKey();
            // Range eventRange = new Range(eventKey, eventKey);
            Range eventRange = new Range(eventKey.getRow());
            HashSet<ByteSequence> cf = new HashSet<ByteSequence>();
            cf.add(eventKey.getColumnFamilyData());
            event.seek(eventRange, cf, true);
            if (event.hasTop()) {
                key = event.getTopKey();
                value = event.getTopValue();
            } else {
                next();
            }
        }
    }
}
 

/**
 * Override this for your specific implementation.
 * 
 * In this specific implementation our row key will be the term, the column family will be the field name, and the column family will be the shard,so we
 * should have the following as our last key
 * 
 * bar FOO:20130101_0
 * 
 * so we should append a null so that we we don't skip shards. similarly, an assumption is made of the key structure within this class.
 * 
 * @param lastKey
 * @param previousRange
 */
public Range buildNextRange(final Key lastKey, final Range previousRange) {
    
    /**
     * This will re-seek the next column family when performing any field expansion.
     */
    Range r = new Range(new Key(lastKey.getRow(), new Text(lastKey.getColumnFamily() + "\u0000\uffff")), true, previousRange.getEndKey(),
                    previousRange.isEndKeyInclusive());
    if (log.isTraceEnabled())
        log.trace(r);
    return r;
    
}