Java Code Examples for org.apache.hadoop.hbase.io.ImmutableBytesWritable#get()

@Test
public void testForVarCharArrayForEvenNumberWithIndex() {
	String[] strArr = new String[5];
	strArr[0] = "abx";
	strArr[1] = "ereref";
	strArr[2] = "random";
	strArr[3] = "random12";
	strArr[4] = "ranzzz";
	PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
			PVarchar.INSTANCE, strArr);
	byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
	ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
       PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
	int offset = ptr.getOffset();
	int length = ptr.getLength();
	byte[] bs = ptr.get();
	byte[] res = new byte[length];
	System.arraycopy(bs, offset, res, 0, length);
	assertEquals("ranzzz", Bytes.toString(res));
}
 

public void testVariableLengthArrayWithElementsMoreThanShortMax() {
    String[] strArr = new String[(2 * Short.MAX_VALUE) + 100]; 
    for(int i = 0 ; i < (2 * Short.MAX_VALUE) + 100; i++ ) {
        String str = "abc";
        for(int j = 0 ; j <= i ;j++) {
            str += "-";
        }
        strArr[i] = str;
    }
    PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
               PVarchar.INSTANCE, strArr);
       byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
       ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
       PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
       int offset = ptr.getOffset();
       int length = ptr.getLength();
       byte[] bs = ptr.get();
       byte[] res = new byte[length];
       System.arraycopy(bs, offset, res, 0, length);
       assertEquals("abc---", Bytes.toString(res));
}
 

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    if (!getStringExpression().evaluate(tuple, ptr)) {
        return false;
    }
    if (ptr.getLength() == 0) {
        ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
        return true;
    }
    byte[] string = ptr.get();
    int offset = ptr.getOffset();
    int length = ptr.getLength();
    
    SortOrder sortOrder = getSortOrder();
    int end = StringUtil.getFirstNonBlankCharIdxFromEnd(string, offset, length, sortOrder);
    if (end == offset - 1) {
        ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
        return true; 
    }
    int head = StringUtil.getFirstNonBlankCharIdxFromStart(string, offset, length, sortOrder);
    ptr.set(string, head, end - head + 1);
    return true;
}
 

/**
 * Decode a vint from the buffer pointed at to by ptr and
 * increment the offset of the ptr by the length of the
 * vint.
 * @param ptr a pointer to a byte array buffer
 * @return the decoded vint value as a long
 */
public static long vlongFromBytes(ImmutableBytesWritable ptr) {
    final byte [] buffer = ptr.get();
    final int offset = ptr.getOffset();
    byte firstByte = buffer[offset];
    int len = WritableUtils.decodeVIntSize(firstByte);
    if (len == 1) {
        ptr.set(buffer, offset+1, ptr.getLength());
        return firstByte;
    }
    long i = 0;
    for (int idx = 0; idx < len-1; idx++) {
        byte b = buffer[offset + 1 + idx];
        i = i << 8;
        i = i | (b & 0xFF);
    }
    ptr.set(buffer, offset+len, ptr.getLength());
    return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
}
 

/**
 * Decode a vint from the buffer pointed at to by ptr and
 * increment the offset of the ptr by the length of the
 * vint.
 * @param ptr a pointer to a byte array buffer
 * @return the decoded vint value as a long
 */
public static long vlongFromBytes(ImmutableBytesWritable ptr) {
    final byte [] buffer = ptr.get();
    final int offset = ptr.getOffset();
    byte firstByte = buffer[offset];
    int len = WritableUtils.decodeVIntSize(firstByte);
    if (len == 1) {
        ptr.set(buffer, offset+1, ptr.getLength());
        return firstByte;
    }
    long i = 0;
    for (int idx = 0; idx < len-1; idx++) {
        byte b = buffer[offset + 1 + idx];
        i = i << 8;
        i = i | (b & 0xFF);
    }
    ptr.set(buffer, offset+len, ptr.getLength());
    return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
}
 

@Override
public KeyValue getValue(byte[] family, byte[] qualifier) {
    ImmutableBytesWritable value = null;
    try {
        value = valueGetter.getLatestValue(new ColumnReference(family, qualifier), ts);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    byte[] rowKey = valueGetter.getRowKey();
    int valueOffset = 0;
    int valueLength = 0;
    byte[] valueBytes = HConstants.EMPTY_BYTE_ARRAY;
    if (value != null) {
        valueBytes = value.get();
        valueOffset = value.getOffset();
        valueLength = value.getLength();
    }
	return new KeyValue(rowKey, 0, rowKey.length, family, 0, family.length, qualifier, 0, qualifier.length, HConstants.LATEST_TIMESTAMP, Type.Put, valueBytes, valueOffset, valueLength);
}
 

@Test
public void testPositionSearchWithVarLengthArrayWithNullValue2() {
    String[] strArr = new String[5];
    strArr[0] = "abx";
    strArr[1] = "ereref";
    strArr[2] = "random";
    strArr[3] = "random12";
    strArr[4] = null;
    PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
            PVarchar.INSTANCE, strArr);
    byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
    ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
    PArrayDataTypeDecoder.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
    int offset = ptr.getOffset();
    int length = ptr.getLength();
    byte[] bs = ptr.get();
    byte[] res = new byte[length];
    System.arraycopy(bs, offset, res, 0, length);
    assertEquals("random", Bytes.toString(res));
}
 

@Test
public void testPositionSearchWithVarLengthArrayWithNullValue2() {
    String[] strArr = new String[5];
    strArr[0] = "abx";
    strArr[1] = "ereref";
    strArr[2] = "random";
    strArr[3] = "random12";
    strArr[4] = null;
    PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
            PVarchar.INSTANCE, strArr);
    byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
    ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
    PArrayDataType.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
    int offset = ptr.getOffset();
    int length = ptr.getLength();
    byte[] bs = ptr.get();
    byte[] res = new byte[length];
    System.arraycopy(bs, offset, res, 0, length);
    assertEquals("random", Bytes.toString(res));
}
 

@Test
public void testPositionSearchWithVarLengthArrayWithNullValue5() {
    String[] strArr = new String[5];
    strArr[0] = "abx";
    strArr[1] = "ereref";
    strArr[2] = "random";
    strArr[3] = null;
    strArr[4] = "ran";
    PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
            PVarchar.INSTANCE, strArr);
    byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
    ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
    PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
    int offset = ptr.getOffset();
    int length = ptr.getLength();
    byte[] bs = ptr.get();
    byte[] res = new byte[length];
    System.arraycopy(bs, offset, res, 0, length);
    assertEquals("", Bytes.toString(res));
}
 

@Test
public void testForVarCharArrayForOneElementArrayWithIndex() {
	String[] strArr = new String[1];
	strArr[0] = "abx";
	PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
			PVarchar.INSTANCE, strArr);
	byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
	ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
	PArrayDataType.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
	int offset = ptr.getOffset();
	int length = ptr.getLength();
	byte[] bs = ptr.get();
	byte[] res = new byte[length];
	System.arraycopy(bs, offset, res, 0, length);
	assertEquals("abx", Bytes.toString(res));
}
 

/**
 * Given an ImmutableBytesWritable, returns the payload part of the argument as an byte array. 
 */
public static byte[] copyKeyBytesIfNecessary(ImmutableBytesWritable ptr) {
    if (ptr.getOffset() == 0 && ptr.getLength() == ptr.get().length) {
        return ptr.get();
    }
    return ptr.copyBytes();
}
 

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    // Starting from the front of the byte, look for all single bytes at the end of the string
    // that is below SPACE_UTF8 (space and control characters) or 0x7f (control chars).
    if (!getStringExpression().evaluate(tuple, ptr)) {
        return false;
    }
    
    if (ptr.getLength() == 0) {
        ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
        return true;
    }
    byte[] string = ptr.get();
    int offset = ptr.getOffset();
    int length = ptr.getLength();
    
    ColumnModifier columnModifier = getStringExpression().getColumnModifier();
    // TODO: when we have ColumnModifier.REVERSE, we'll need to trim from the end instead of
    // the beginning (just delegate to RTrimFunction or replace from ExpressionCompiler instead?)
    int i = StringUtil.getFirstNonBlankCharIdxFromStart(string, offset, length, columnModifier);
    if (i == offset + length) {
        ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
        return true;
    }
    
    ptr.set(string, i, offset + length - i);
    return true;
}
 

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    if (!children.get(0).evaluate(tuple, ptr)) {
        return false;
    }
    if (ptr.getLength() == 0) { // null comparison evals to null
        return true;
    }
    byte[] lhsBytes = ptr.get();
    int lhsOffset = ptr.getOffset();
    int lhsLength = ptr.getLength();
    PDataType lhsDataType = children.get(0).getDataType();
    SortOrder lhsSortOrder = children.get(0).getSortOrder();
    
    if (!children.get(1).evaluate(tuple, ptr)) {
        return false;
    }
    if (ptr.getLength() == 0) { // null comparison evals to null
        return true;
    }
    
    byte[] rhsBytes = ptr.get();
    int rhsOffset = ptr.getOffset();
    int rhsLength = ptr.getLength();
    PDataType rhsDataType = children.get(1).getDataType();
    SortOrder rhsSortOrder = children.get(1).getSortOrder();   
    if (rhsDataType == PChar.INSTANCE) {
        rhsLength = StringUtil.getUnpaddedCharLength(rhsBytes, rhsOffset, rhsLength, rhsSortOrder);
    }
    if (lhsDataType == PChar.INSTANCE) {
        lhsLength = StringUtil.getUnpaddedCharLength(lhsBytes, lhsOffset, lhsLength, lhsSortOrder);
    }
    
    
    int comparisonResult = lhsDataType.compareTo(lhsBytes, lhsOffset, lhsLength, lhsSortOrder, 
            rhsBytes, rhsOffset, rhsLength, rhsSortOrder, rhsDataType);
    ptr.set(ByteUtil.compare(op, comparisonResult) ? PDataType.TRUE_BYTES : PDataType.FALSE_BYTES);
    return true;
}
 

public static KeyValue newKeyValue(ImmutableBytesWritable key, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key.get(), key.getOffset(), key.getLength(),
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 

@Override
protected void map(ImmutableBytesWritable key, ImmutableBytesWritable value, Context context) throws IOException, InterruptedException {

    KeyValue kv = new KeyValue(key.get(), key.getOffset(), key.getLength(), //
            IIDesc.HBASE_FAMILY_BYTES, 0, IIDesc.HBASE_FAMILY_BYTES.length, //
            IIDesc.HBASE_QUALIFIER_BYTES, 0, IIDesc.HBASE_QUALIFIER_BYTES.length, //
            timestamp, Type.Put, //
            value.get(), value.getOffset(), value.getLength());

    context.write(key, kv);
}
 

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    Expression child = children.get(0);
    if (!child.evaluate(tuple, ptr)) {
        return false;
    }
    if (ptr.getLength() == 0) {
        return true;
    }
    byte serializedByte = ptr.get()[ptr.getOffset()];
    PIndexState indexState = PIndexState.fromSerializedValue(serializedByte);
    ptr.set(indexState.toBytes());
    return true;
}
 

public static byte[] copyBytesIfNecessary(ImmutableBytesWritable ptr) {
  if (ptr.getOffset() == 0 && ptr.getLength() == ptr.get().length) {
    return ptr.get();
  }
  return ptr.copyBytes();
}
 

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    AbstractBasePattern pattern = this.pattern;
    if (pattern == null) {
        Expression e = getPatternStrExpression();
        if (!e.evaluate(tuple, ptr)) {
            return false;
        }
        if (ptr.getLength()==0) {
            return true;
        }
        String patternStr = (String) TYPE.toObject(ptr, e.getDataType(), e.getSortOrder());
        if (patternStr == null) {
            return false;
        } else {
            pattern = compilePatternSpec(patternStr);
        }
    }

    byte[] rStrBytes = this.rStrBytes;
    int rStrOffset = this.rStrOffset, rStrLen = this.rStrLen;
    if (rStrBytes == null) {
        Expression replaceStrExpression = getReplaceStrExpression();
        if (!replaceStrExpression.evaluate(tuple, ptr)) {
            return false;
        }
        if (ptr.getLength()==0) {
            return true;
        }
        TYPE.coerceBytes(ptr, TYPE, replaceStrExpression.getSortOrder(), SortOrder.ASC);
        rStrBytes = ptr.get();
        rStrOffset = ptr.getOffset();
        rStrLen = ptr.getLength();
    }

    Expression sourceStrExpression = getSourceStrExpression();
    if (!sourceStrExpression.evaluate(tuple, ptr)) {
        return false;
    }
    if (ptr.getLength()==0) {
        return true;
    }
    TYPE.coerceBytes(ptr, TYPE, sourceStrExpression.getSortOrder(), SortOrder.ASC);

    pattern.replaceAll(ptr, rStrBytes, rStrOffset, rStrLen);
    return true;
}
 

public ImmutableBytesPtr(ImmutableBytesWritable ibw) {
    super(ibw.get(), ibw.getOffset(), ibw.getLength());
    hashCode = super.hashCode();
}
 

@Override
protected void map(ImmutableBytesWritable key, Result value, Context output) throws IOException, InterruptedException {
    byte region = key.get()[key.getOffset()];
    List<Statement> stmts = null;
    int hashShift;
    if (region < HalyardTableUtils.CSPO_PREFIX) {
        hashShift = 1;
    } else {
        hashShift = HalyardTableUtils.KEY_SIZE + 1;
        if (!matchAndCopyKey(key.get(), key.getOffset() + 1, lastCtxFragment) || region != lastRegion) {
            cleanup(output);
            stmts = HalyardTableUtils.parseStatements(value, ssf);
            graph = (IRI) stmts.get(0).getContext();
        }
        if (update && region == HalyardTableUtils.CSPO_PREFIX) {
            if (Arrays.equals(statsContextHash, lastCtxFragment)) {
                if (sail == null) {
                    Configuration conf = output.getConfiguration();
                    sail = new HBaseSail(conf, conf.get(SOURCE), false, 0, true, 0, null, null);
                    sail.initialize();
                }
                if (stmts == null) {
                    stmts = HalyardTableUtils.parseStatements(value, ssf);
                }
                for (Statement st : stmts) {
                    if (statsContext.equals(st.getContext()) && matchingGraphContext(st.getSubject())) {
                        sail.removeStatement(null, st.getSubject(), st.getPredicate(), st.getObject(), st.getContext());
                        removed++;
                    }
                }
                lastRegion = region;
                return; //do no count removed statements
            }
        }
    }
    boolean hashChange = !matchAndCopyKey(key.get(), key.getOffset() + hashShift, lastKeyFragment) || region != lastRegion || lastGraph != graph;
    if (hashChange) {
        cleanupSubset(output);
        if (stmts == null) {
            stmts = HalyardTableUtils.parseStatements(value, ssf);
        }
        Statement stmt = stmts.get(0);
        switch (region) {
            case HalyardTableUtils.SPO_PREFIX:
            case HalyardTableUtils.CSPO_PREFIX:
                distinctSubjects++;
                Resource subj = stmt.getSubject();
                if (subj instanceof IRI) {
                    distinctIRIReferenceSubjects++;
                } else {
                    distinctBlankNodeSubjects++;
                }
                subsetType = VOID_EXT.SUBJECT;
                subsetId = subj;
                break;
            case HalyardTableUtils.POS_PREFIX:
            case HalyardTableUtils.CPOS_PREFIX:
                properties++;
                subsetType = VOID.PROPERTY;
                subsetId = stmt.getPredicate();
                break;
            case HalyardTableUtils.OSP_PREFIX:
            case HalyardTableUtils.COSP_PREFIX:
                distinctObjects++;
                Value obj = stmt.getObject();
                if (obj instanceof IRI) {
                    distinctIRIReferenceObjects++;
                } else if (obj instanceof BNode) {
                    distinctBlankNodeObjects++;
                } else {
                    distinctLiterals++;
                }
                subsetType = VOID_EXT.OBJECT;
                subsetId = obj;
                break;
            default:
                throw new IOException("Unknown region #" + region);
        }
    }
    switch (region) {
        case HalyardTableUtils.SPO_PREFIX:
        case HalyardTableUtils.CSPO_PREFIX:
            triples += value.rawCells().length;
            break;
        case HalyardTableUtils.POS_PREFIX:
        case HalyardTableUtils.CPOS_PREFIX:
            if (Arrays.equals(TYPE_HASH, lastKeyFragment) && (!matchAndCopyKey(key.get(), key.getOffset() + hashShift + HalyardTableUtils.KEY_SIZE, lastClassFragment) || hashChange)) {
                classes++;
            }
            break;
        default:
    }
    subsetCounter += value.rawCells().length;
    setCounter += value.rawCells().length;
    lastRegion = region;
    lastGraph = graph;
    if ((counter++ % 100000) == 0) {
        output.setStatus(MessageFormat.format("reg:{0} {1} t:{2} s:{3} p:{4} o:{5} c:{6} r:{7}", region, counter, triples, distinctSubjects, properties, distinctObjects, classes, removed));
    }
}