Java Code Examples for org.apache.accumulo.core.data.Key#getTimestamp()

/**
 * Given a Key to consume, update any necessary counters etc.
 *
 * @param key
 */
private void consume(Key key) {
    
    if (log.isTraceEnabled()) {
        log.trace("consume, key: " + key);
    }
    
    // update the visibility set
    MutableInt counter = this.currentVisibilityCounts.get(key.getColumnVisibility());
    if (counter == null) {
        this.currentVisibilityCounts.put(key.getColumnVisibility(), new MutableInt(1));
    } else {
        counter.increment();
    }
    
    // update current count
    this.count += 1;
    
    // set most recent timestamp
    this.maxTimeStamp = (this.maxTimeStamp > key.getTimestamp()) ? maxTimeStamp : key.getTimestamp();
}
 

protected Key getEndKey(JexlNode node) {
    Key endKey = rangeLimiter.getEndKey();
    String identifier = JexlASTHelper.getIdentifier(node);
    Object objValue = JexlASTHelper.getLiteralValue(node);
    String value = null == objValue ? "null" : objValue.toString();
    
    StringBuilder builder = new StringBuilder("fi");
    builder.append(NULL_DELIMETER).append(identifier);
    
    Text cf = new Text(builder.toString());
    
    builder = new StringBuilder(value);
    
    builder.append(NULL_DELIMETER).append(endKey.getColumnFamily());
    Text cq = new Text(builder.toString());
    
    return new Key(endKey.getRow(), cf, cq, endKey.getTimestamp());
}
 

private String keyToString(Key key) {
    if (key == null) {
        return "null";
    }
    StringBuilder sb = new StringBuilder();
    appendText(sb, key.getRow());
    if (key.getColumnFamily() != null && key.getColumnFamily().getLength() > 0) {
        sb.append(":");
        appendText(sb, key.getColumnFamily());
    }
    if (key.getColumnQualifier() != null && key.getColumnQualifier().getLength() > 0) {
        sb.append(":");
        appendText(sb, key.getColumnQualifier());
    }
    if (key.getColumnVisibility() != null && key.getColumnVisibility().getLength() > 0) {
        sb.append(":");
        appendText(sb, key.getColumnVisibility());
    }
    if (key.getTimestamp() != Long.MAX_VALUE) {
        sb.append(":");
        sb.append(key.getTimestamp());
    }
    return sb.toString();
}
 

@Override
public Key getReturnKey(Key k) {
    // If we were using column visibility, then we would get the merged visibility here and use it in the key.
    // Remove the COLQ from the key and use the combined visibility
    Key r = new Key(k.getRowData().getBackingArray(), k.getColumnFamilyData().getBackingArray(), EMPTY_BYTE, k.getColumnVisibility().getBytes(),
        k.getTimestamp(), k.isDeleted(), false);
    return r;
}
 

private KeyValue replaceEventUidInCF(KeyValue keyValue, int partIndex, boolean startKey, boolean startKeyInclusive, boolean endKey, boolean endKeyInclusive) {
    Key key = keyValue.getKey();
    String[] replacement = replaceUid(key.getColumnFamily().toString(), partIndex, startKey, startKeyInclusive, endKey, endKeyInclusive);
    // if no change in the uid, then return the original key
    if (replacement == null) {
        return keyValue;
    }
    
    Key newKey = new Key(key.getRow(), new Text(replacement[CQ_INDEX]), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp());
    return new KeyValue(newKey, replacement[ORG_UID_INDEX].getBytes());
}
 

/********************** Helper methods ***********************/
private KeyValue replaceEventUidInCQ(KeyValue keyValue, int partIndex, boolean startKey, boolean startKeyInclusive, boolean endKey, boolean endKeyInclusive) {
    Key key = keyValue.getKey();
    String[] replacement = replaceUid(key.getColumnQualifier().toString(), partIndex, startKey, startKeyInclusive, endKey, endKeyInclusive);
    // if no change in the uid, then return the original key
    if (replacement == null) {
        return keyValue;
    }
    
    Key newKey = new Key(key.getRow(), key.getColumnFamily(), new Text(replacement[CQ_INDEX]), key.getColumnVisibility(), key.getTimestamp());
    return new KeyValue(newKey, replacement[ORG_UID_INDEX].getBytes());
}
 

public void aggregate(Key key, Value val) {
    uniqueVisibilities.add(key.getColumnVisibility());
    count += Long.parseLong(new String(val.get()));
    if (maxTimestamp < key.getTimestamp()) {
        maxTimestamp = key.getTimestamp();
    }
}
 

public static Key fieldIndexKeyToEventKey(Key key) {
    // field index key is shardId : fi\0fieldName : fieldValue\0datatype\0uid
    // event key is shardId : dataType\0uid : fieldName\0fieldValue
    String cf = key.getColumnFamily().toString();
    String cq = key.getColumnQualifier().toString();
    int cqNullIndex = cq.indexOf('\0');
    return new Key(key.getRow(), new Text(cq.substring(cqNullIndex + 1)), new Text(cf.substring(3) + '\0' + cq.substring(0, cqNullIndex)),
                    key.getColumnVisibility(), key.getTimestamp());
}
 

/**
 * Required by the {@code FilterRule} interface. This method returns a {@code boolean} value indicating whether or not to allow the {@code (Key, Value)}
 * pair through the rule. A value of {@code true} indicates that he pair should be passed onward through the {@code Iterator} stack, and {@code false}
 * indicates that the {@code (Key, Value)} pair should not be passed on.
 * 
 * <p>
 * If the value provided in the paramter {@code k} does not match the REGEX pattern specified in this filter's configuration options, then a value of
 * {@code true} is returned.
 * 
 * @param k
 *            {@code Key} object containing the row, column family, and column qualifier.
 * @param v
 *            {@code Value} object containing the value corresponding to the {@code Key: k}
 * @return {@code boolean} value indicating whether or not to allow the {@code Key, Value} through the {@code Filter}.
 */
@Override
public boolean accept(AgeOffPeriod period, Key k, Value v) {
    // Keep the pair if its date is after the cutoff date
    boolean dtFlag = false;
    ruleApplied = false;
    
    if (this.pattern == null) // stringLiteral is not being used
    {
        if (log.isTraceEnabled())
            log.trace("dataTypeRegex == null");
        dtFlag = true;
    } else {
        String keyField = getKeyField(k, v);
        Matcher matcher = pattern.matcher(keyField);
        if (matcher.find()) {
            long timeStamp = k.getTimestamp();
            dtFlag = timeStamp > period.getCutOffMilliseconds();
            if (log.isTraceEnabled()) {
                log.trace("timeStamp = " + timeStamp);
                log.trace("timeStamp = " + period.getCutOffMilliseconds());
                log.trace("timeStamp as Date = " + new Date(timeStamp));
            }
            ruleApplied = true;
        } else {
            if (log.isTraceEnabled()) {
                log.trace(keyField + "did not match the pattern:" + this.patternStr);
            }
            dtFlag = true;
        }
        
    }
    if (log.isTraceEnabled()) {
        log.trace("dtFlag = " + dtFlag);
    }
    return dtFlag;
}
 

@Override
protected void map(Key key, Value value, Context context) throws IOException, InterruptedException {
    key.getRow(holder);
    String name = holder.toString().substring(12);
    long loadTime = key.getTimestamp();
    String outRow = DateHelper.format(new Date(loadTime));
    Matcher m = radixRegex.matcher(name);
    key.getColumnQualifier(holder);
    if (m.matches()) {
        context.write(makeKey(outRow, "GROOMER_RLABEL_FILES", m.group(1), defaultVisibility), makeValue("1"));
        context.write(makeKey(outRow, "GROOMER_RLABEL_BYTES", m.group(1), defaultVisibility), makeValue(holder.toString()));
    }
}
 

@Override
public Key getTopKey() {
    Key topKey = super.getTopKey();

    long timestamp = reverseTimestampToNormalTime(Long.parseLong(topKey.getRow().toString()));

    Key retKey = new Key(new Text(truncatedReverseTimestamp(timestamp, bucketSize).toString()),
            topKey.getColumnFamily(), topKey.getColumnQualifier(),
            new Text(topKey.getColumnVisibility().toString()), topKey.getTimestamp());

    return retKey;
}
 

/**
 * Required by the {@code FilterRule} interface. This method returns a {@code boolean} value indicating whether or not to allow the {@code (Key, Value)}
 * pair through the rule. A value of {@code true} indicates that he pair should be passed onward through the {@code Iterator} stack, and {@code false}
 * indicates that the {@code (Key, Value)} pair should not be passed on.
 * 
 * <p>
 * If the value provided in the paramter {@code k} does not match the STRING pattern specified in this filter's configuration options, then a value of
 * {@code true} is returned.
 * 
 * @param k
 *            {@code Key} object containing the row, column family, and column qualifier.
 * @param v
 *            {@code Value} object containing the value corresponding to the {@code Key: k}
 * @return {@code boolean} value indicating whether or not to allow the {@code Key, Value} through the {@code Filter}.
 */
@Override
public boolean accept(AgeOffPeriod period, Key k, Value v) {
    // Keep the pair if its date is after the cutoff date
    boolean dtFlag = false;
    ruleApplied = false;
    
    if (this.patternBytes == null) // patterns are not being used
    {
        log.trace("patternBytes == null");
        dtFlag = true;
    } else {
        if (hasToken(k, v, patternBytes)) {
            long timeStamp = k.getTimestamp();
            dtFlag = timeStamp > period.getCutOffMilliseconds();
            if (log.isTraceEnabled()) {
                log.trace("timeStamp = " + timeStamp);
                log.trace("timeStamp = " + period.getCutOffMilliseconds());
                log.trace("timeStamp as Date = " + new Date(timeStamp));
            }
            ruleApplied = true;
        } else {
            if (log.isTraceEnabled()) {
                log.trace("did not match the patterns:" + toString(patternBytes));
            }
            // if the pattern did not match anything, then go ahead and accept this record
            dtFlag = true;
        }
        
    }
    if (log.isTraceEnabled()) {
        log.trace("dtFlag = " + dtFlag);
    }
    return dtFlag;
}
 

@Override
public boolean accept(Key key, Value value) {
    long ts = key.getTimestamp();
    Timestamp timestamp = timestamps.get(key.getRow());
    if (timestamp == null) {
        return true;
    }
    if (timestamp.startTimestamp != null) {
        if (timestamp.startInclusive) {
            if (ts < timestamp.startTimestamp) {
                return false;
            }
        } else {
            if (ts <= timestamp.startTimestamp) {
                return false;
            }
        }
    }
    if (timestamp.endTimestamp != null) {
        if (timestamp.endInclusive) {
            if (ts > timestamp.endTimestamp) {
                return false;
            }
        } else {
            if (ts >= timestamp.endTimestamp) {
                return false;
            }
        }
    }
    return true;
}
 

/**
 * This method returns a {@code boolean} value indicating whether or not to allow the {@code (Key, Value)} pair through the filter. A value of {@code true}
 * indicates that he pair should be passed onward through the {@code Iterator} stack, and {@code false} indicates that the {@code (Key, Value)} pair should
 * not be passed on.
 *
 * @param k
 *            {@code Key} object containing the row, column family, and column qualifier.
 * @param v
 *            {@code Value} object containing the value corresponding to the {@code Key: k}
 * @return {@code boolean} value indicating whether or not to allow the {@code Key, Value} through the {@code Filter}.
 */
@Override
public boolean accept(Key k, Value v) {
    
    // if disabled, simple pass through
    if (this.disabled)
        return true;
    
    // short circuit check
    long timeStamp = k.getTimestamp();
    if (timeStamp > this.shortCircuitDateMillis)
        return true;
    
    boolean acceptFlag = false;
    boolean filterRuleApplied = false;
    
    Iterator<AppliedRule> iter = this.filterList.iterator();
    
    while ((!filterRuleApplied) && iter.hasNext()) {
        AppliedRule filter = iter.next();
        acceptFlag = filter.accept(k, v);
        filterRuleApplied = filter.isFilterRuleApplied();
    }
    
    // We went through all of the defined filter rules
    // and none were used, let's apply the default TTL
    if (!filterRuleApplied) {
        acceptFlag = timeStamp > this.cutOffDateMillis;
    }
    if (log.isTraceEnabled()) {
        log.trace("acceptFlag = " + acceptFlag);
    }
    return acceptFlag;
    
}
 

private Key addKeyCount(Key key) {
    resultCount.getAndIncrement();
    return new Key(key.getRow(), new Text(NumericalEncoder.encode(Long.toString(resultCount.get())) + '\0' + key.getColumnFamily()),
                    key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp());
}
 

protected long parseTimestamp(Key k, Value v) {
    return k.getTimestamp();
}
 

@Override
public boolean accept(Key k, Value v) {
    long diff = startTime - k.getTimestamp();
    return !(diff > timeRange || diff < 0);
}
 

/**
 * For each index row in the lastN store, grab the associated getAttributes (further down in the tablet) and construct
 * the entry to be returned.
 *
 * @return
 */
@Override
public Value getTopValue() {

    if (hasTop()) {

        Key topKey = getTopKey();
        Value topVal = super.getTopValue();
        String entryId = new String(topVal.get());

        Key startRangeKey = new Key(topKey.getRow(), new Text(END_BYTE + entryId));
        Key stopRangeKey = new Key(topKey.getRow(), new Text(END_BYTE + entryId + END_BYTE));

        Range range = new Range(startRangeKey, stopRangeKey);

        long timestamp = 0;

        try {
            sourceItr.seek(range, Collections.<ByteSequence>emptyList(), false);

            Collection<Attribute> attributes = new ArrayList<Attribute>();
            while (sourceItr.hasTop()) {

                Key nextKey = sourceItr.getTopKey();
                sourceItr.next();

                if (!nextKey.getColumnFamily().toString().endsWith(entryId)) {
                    break;
                }

                String[] keyValueDatatype = nextKey.getColumnQualifier().toString().split(NULL_BYTE);

                if (keyValueDatatype.length == 3) {

                    String vis = nextKey.getColumnVisibility().toString();

                    Attribute attribute = new Attribute(
                            keyValueDatatype[0],
                            typeRegistry.decode(keyValueDatatype[2], keyValueDatatype[1]),
                            setVisibility(new HashMap<String, String>(1), vis)
                    );


                    attributes.add(attribute);
                    timestamp = nextKey.getTimestamp();
                }
            }

            int oneByte = entryId.indexOf(ONE_BYTE);
            String finalType = entryId.substring(0, oneByte);
            String finalId = entryId.substring(oneByte+1, entryId.length());

            EventBuilder entry = EventBuilder.create(finalType, finalId, timestamp);

            if (attributes.size() > 0)
                entry.attrs(attributes);

            writable.set(entry.build());
            return new Value(serialize(writable));

        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    return new Value("".getBytes());
}
 

@Override
public boolean accept(Key k, Value v) {

    // If less than any configured ageoff, then keep it
    if ((this.currentTime - k.getTimestamp()) < this.minAgeOff) {
        return true;
    }
    // If greater than any configured ageoff, then drop it
    if ((this.currentTime - k.getTimestamp()) > this.maxAgeOff) {
        return false;
    }
    // There is a high probability that this key will have
    // the same metric name as the last key.
    ByteSequence rowData = k.getRowData();
    int rowStart = rowData.offset();
    int i = 0;
    if (null != prevMetricBytes && (rowData.length() >= (rowStart + prevMetricBytes.length + 1))
            && (rowData.byteAt(rowStart + prevMetricBytes.length + 1) == 0x00)) {
        // Double check metric name is the same
        boolean same = true;
        for (; i < prevMetricBytes.length; i++) {
            if (prevMetricBytes[i] != rowData.byteAt(rowStart + i)) {
                same = false;
                break;
            }
        }
        if (same) {
            if ((currentTime - k.getTimestamp()) > prevAgeOff)
                return false;
            return true;
        }
    }

    // Metric name is different or prev information is not set

    // We have not found the null byte up to this point.
    // Keep scanning for the null byte
    int y = rowStart + i;
    for (; y < rowData.length(); y++) {
        byte b = rowData.byteAt(y);
        if (b == 0x00) {
            break;
        }
    }
    byte[] metricName = new byte[(y - rowStart)];
    System.arraycopy(rowData.getBackingArray(), rowStart, metricName, 0, (y - rowStart));

    prevMetricBytes = metricName;
    prevAgeOff = ageoffs.get(new String(prevMetricBytes, UTF_8));

    if (null == prevAgeOff) {
        // no specific ageoff for this metric name, use default
        prevAgeOff = defaultAgeOff;
    }
    if (currentTime - k.getTimestamp() > prevAgeOff)
        return false;
    return true;
}
 

/**
 * When skipKey is false, we produce the key based upon k, removing any data type
 * 
 * When skipKey is true, we will produce a key producing a skipkey from the root key. This will be helpful when we are being torn down.
 * 
 * @param k
 * @return
 */
public static Key makeRootKey(Key k) {
    ByteSequence cq = k.getColumnQualifierData();
    ByteSequence strippedCq = cq.subSequence(0, lastNull(cq));
    final ByteSequence row = k.getRowData(), cf = k.getColumnFamilyData(), cv = k.getColumnVisibilityData();
    return new Key(row.getBackingArray(), row.offset(), row.length(), cf.getBackingArray(), cf.offset(), cf.length(), strippedCq.getBackingArray(),
                    strippedCq.offset(), strippedCq.length(), cv.getBackingArray(), cv.offset(), cv.length(), k.getTimestamp());
}