Java Code Examples for org.apache.cassandra.db.marshal.UTF8Type#instance()

public static AbstractType getCassandraType(PrimitiveObjectInspector oi) {
  switch (oi.getPrimitiveCategory()) {
  case BOOLEAN:
    return BooleanType.instance;
  case INT:
    return Int32Type.instance;
  case LONG:
    return LongType.instance;
  case FLOAT:
    return FloatType.instance;
  case DOUBLE:
    return DoubleType.instance;
  case STRING:
    return UTF8Type.instance;
  case BYTE:
  case SHORT:
  case BINARY:
    return BytesType.instance;
  case TIMESTAMP:
    return DateType.instance;
  default:
    throw new RuntimeException("Hive internal error.");

  }
}
 

private void validateAssignableTo(String keyspace, ColumnSpecification receiver) throws InvalidRequestException
{
    if (!(receiver.type instanceof UserType))
        throw new InvalidRequestException(String.format("Invalid user type literal for %s of type %s", receiver, receiver.type.asCQL3Type()));

    UserType ut = (UserType)receiver.type;
    for (int i = 0; i < ut.size(); i++)
    {
        ColumnIdentifier field = new ColumnIdentifier(ut.fieldName(i), UTF8Type.instance);
        Term.Raw value = entries.get(field);
        if (value == null)
            continue;

        ColumnSpecification fieldSpec = fieldSpecOf(receiver, i);
        if (!value.isAssignableTo(keyspace, fieldSpec))
            throw new InvalidRequestException(String.format("Invalid user type literal for %s: field %s is not of type %s", receiver, field, fieldSpec.type.asCQL3Type()));
    }
}
 

@Test
public void testSatisfiedByWithMultipleTerms()
{
    final ByteBuffer comment = UTF8Type.instance.decompose("comment");
    final ColumnFamilyStore store = Keyspace.open("sasecondaryindex").getColumnFamilyStore("saindexed1");
    final IPartitioner<?> partitioner = StorageService.getPartitioner();

    ColumnFamily cf = ArrayBackedSortedColumns.factory.create(store.metadata);
    cf.addColumn(new Column(comment, UTF8Type.instance.decompose("software engineer is working on a project"), System.currentTimeMillis()));

    Operation.Builder builder = new Operation.Builder(OperationType.AND, UTF8Type.instance, controller,
                                        new IndexExpression(comment, IndexOperator.EQ, UTF8Type.instance.decompose("eng is a work")));
    Operation op = builder.complete();

    Assert.assertTrue(op.satisfiedBy(new Row(partitioner.decorateKey(UTF8Type.instance.decompose("key1")), cf), null, false));

    builder = new Operation.Builder(OperationType.AND, UTF8Type.instance, controller,
                                        new IndexExpression(comment, IndexOperator.EQ, UTF8Type.instance.decompose("soft works fine")));
    op = builder.complete();

    Assert.assertTrue(op.satisfiedBy(new Row(partitioner.decorateKey(UTF8Type.instance.decompose("key1")), cf), null, false));
}
 

private CFMetaData addTestCF(String ks, String cf, String comment)
{
    CFMetaData newCFMD = new CFMetaData(ks, cf, ColumnFamilyType.Standard, new SimpleDenseCellNameType(UTF8Type.instance));
    newCFMD.comment(comment)
           .readRepairChance(0.0);

    return newCFMD;
}
 

@Test
public void testText() {
    DataType textType = DataType.text();
    AbstractType<?> convertedType = CassandraTypeConverter.convert(textType);

    UTF8Type expectedType = UTF8Type.instance;

    Assert.assertEquals(expectedType, convertedType);
}
 

@Test
public void testSerializedSize() throws IOException
{
    final RowIndexEntry simple = new RowIndexEntry(123);

    DataOutputBuffer buffer = new DataOutputBuffer();
    RowIndexEntry.Serializer serializer = new RowIndexEntry.Serializer(new SimpleDenseCellNameType(UTF8Type.instance));

    serializer.serialize(simple, buffer);

    Assert.assertEquals(buffer.getLength(), serializer.serializedSize(simple));

    buffer = new DataOutputBuffer();
    ColumnFamily cf = ArrayBackedSortedColumns.factory.create("Keyspace1", "Standard1");
    ColumnIndex columnIndex = new ColumnIndex.Builder(cf, ByteBufferUtil.bytes("a"), new DataOutputBuffer())
    {{
        int idx = 0, size = 0;
        Cell column;
        do
        {
            column = new BufferCell(CellNames.simpleDense(ByteBufferUtil.bytes("c" + idx++)), ByteBufferUtil.bytes("v"), FBUtilities.timestampMicros());
            size += column.serializedSize(new SimpleDenseCellNameType(UTF8Type.instance), TypeSizes.NATIVE);

            add(column);
        }
        while (size < DatabaseDescriptor.getColumnIndexSize() * 3);

    }}.build();

    RowIndexEntry withIndex = RowIndexEntry.create(0xdeadbeef, DeletionTime.LIVE, columnIndex);

    serializer.serialize(withIndex, buffer);
    Assert.assertEquals(buffer.getLength(), serializer.serializedSize(withIndex));
}
 

/**
 * Builds a new {@link GeoShapeMapper}.
 */
@JsonCreator
public GeoShapeMapper(@JsonProperty("max_levels") Integer maxLevels) {
    super(new AbstractType<?>[]{AsciiType.instance, UTF8Type.instance});
    this.maxLevels = maxLevels == null ? DEFAULT_MAX_LEVELS : maxLevels;
    this.grid = new GeohashPrefixTree(spatialContext, this.maxLevels);
}
 

@Test
public void testSuperBlocks() throws Exception
{
    Map<ByteBuffer, TokenTreeBuilder> terms = new HashMap<>();
    terms.put(UTF8Type.instance.decompose("1234"), keyBuilder(1L, 2L));
    terms.put(UTF8Type.instance.decompose("2345"), keyBuilder(3L, 4L));
    terms.put(UTF8Type.instance.decompose("3456"), keyBuilder(5L, 6L));
    terms.put(UTF8Type.instance.decompose("4567"), keyBuilder(7L, 8L));
    terms.put(UTF8Type.instance.decompose("5678"), keyBuilder(9L, 10L));

    OnDiskIndexBuilder builder = new OnDiskIndexBuilder(UTF8Type.instance, Int32Type.instance, OnDiskIndexBuilder.Mode.SPARSE);
    for (Map.Entry<ByteBuffer, TokenTreeBuilder> entry : terms.entrySet())
        addAll(builder, entry.getKey(), entry.getValue());

    File index = File.createTempFile("on-disk-sa-try-superblocks", ".db");
    index.deleteOnExit();

    builder.finish(index);

    OnDiskIndex onDisk = new OnDiskIndex(index, Int32Type.instance, new KeyConverter());
    OnDiskIndex.OnDiskSuperBlock superBlock = onDisk.dataLevel.getSuperBlock(0);
    Iterator<Token> iter = superBlock.iterator();

    Long lastToken = null;
    while (iter.hasNext())
    {
        Token token = iter.next();

        if (lastToken != null)
            Assert.assertTrue(lastToken.compareTo(token.get()) < 0);

        lastToken = token.get();
    }
}
 

@Test
public void testDescriptor() throws Exception
{
    final Map<ByteBuffer, Pair<DecoratedKey, Long>> data = new HashMap<ByteBuffer, Pair<DecoratedKey, Long>>()
    {{
            put(Int32Type.instance.decompose(5), Pair.create(keyAt(1L), 1L));
    }};

    OnDiskIndexBuilder builder1 = new OnDiskIndexBuilder(UTF8Type.instance, Int32Type.instance, OnDiskIndexBuilder.Mode.ORIGINAL);
    OnDiskIndexBuilder builder2 = new OnDiskIndexBuilder(UTF8Type.instance, Int32Type.instance, OnDiskIndexBuilder.Mode.ORIGINAL);
    for (Map.Entry<ByteBuffer, Pair<DecoratedKey, Long>> e : data.entrySet())
    {
        DecoratedKey key = e.getValue().left;
        Long position = e.getValue().right;

        builder1.add(e.getKey(), key, position);
        builder2.add(e.getKey(), key, position);
    }

    File index1 = File.createTempFile("on-disk-sa-int", "db");
    File index2 = File.createTempFile("on-disk-sa-int2", "db");
    index1.deleteOnExit();
    index2.deleteOnExit();

    builder1.finish(index1);
    builder2.finish(new Descriptor(Descriptor.VERSION_AA), index2);

    OnDiskIndex onDisk1 = new OnDiskIndex(index1, Int32Type.instance, new KeyConverter());
    OnDiskIndex onDisk2 = new OnDiskIndex(index2, Int32Type.instance, new KeyConverter());

    ByteBuffer number = Int32Type.instance.decompose(5);

    Assert.assertEquals(Collections.singleton(data.get(number).left), convert(onDisk1.search(expressionFor(Int32Type.instance, number))));
    Assert.assertEquals(Collections.singleton(data.get(number).left), convert(onDisk2.search(expressionFor(Int32Type.instance, number))));

    Assert.assertEquals(onDisk1.descriptor.version.version, Descriptor.CURRENT_VERSION);
    Assert.assertEquals(onDisk2.descriptor.version.version, Descriptor.VERSION_AA);
}
 

@Test
public void testNotEqualsQueryForNumbers() throws Exception
{
    final Map<ByteBuffer, TokenTreeBuilder> data = new HashMap<ByteBuffer, TokenTreeBuilder>()
    {{
            put(Int32Type.instance.decompose(5),  keyBuilder(1L));
            put(Int32Type.instance.decompose(7),  keyBuilder(2L));
            put(Int32Type.instance.decompose(1),  keyBuilder(3L));
            put(Int32Type.instance.decompose(3),  keyBuilder(1L, 4L));
            put(Int32Type.instance.decompose(8),  keyBuilder(8L, 6L));
            put(Int32Type.instance.decompose(10), keyBuilder(5L));
            put(Int32Type.instance.decompose(6),  keyBuilder(7L));
            put(Int32Type.instance.decompose(4),  keyBuilder(9L, 10L));
            put(Int32Type.instance.decompose(0),  keyBuilder(11L, 12L, 1L));
    }};

    OnDiskIndexBuilder builder = new OnDiskIndexBuilder(UTF8Type.instance, Int32Type.instance, OnDiskIndexBuilder.Mode.ORIGINAL);
    for (Map.Entry<ByteBuffer, TokenTreeBuilder> e : data.entrySet())
        addAll(builder, e.getKey(), e.getValue());

    File index = File.createTempFile("on-disk-sa-except-int-test", "db");
    index.deleteOnExit();

    builder.finish(index);

    OnDiskIndex onDisk = new OnDiskIndex(index, Int32Type.instance, new KeyConverter());

    Assert.assertEquals(convert(1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12), convert(onDisk.search(expressionForNot(0, 10, 1))));
    Assert.assertEquals(convert(1, 2, 4, 5, 7, 9, 10, 11, 12), convert(onDisk.search(expressionForNot(0, 10, 1, 8))));
    Assert.assertEquals(convert(1, 2, 4, 5, 7, 11, 12), convert(onDisk.search(expressionForNot(0, 10, 1, 8, 4))));

    onDisk.close();
}
 

@Test
public void testMultiSuffixMatches() throws Exception
{
    OnDiskIndexBuilder builder = new OnDiskIndexBuilder(UTF8Type.instance, UTF8Type.instance, OnDiskIndexBuilder.Mode.SUFFIX)
    {{
            addAll(this, UTF8Type.instance.decompose("Eliza"), keyBuilder(1L, 2L));
            addAll(this, UTF8Type.instance.decompose("Elizabeth"), keyBuilder(3L, 4L));
            addAll(this, UTF8Type.instance.decompose("Aliza"), keyBuilder(5L, 6L));
            addAll(this, UTF8Type.instance.decompose("Taylor"), keyBuilder(7L, 8L));
            addAll(this, UTF8Type.instance.decompose("Pavel"), keyBuilder(9L, 10L));
    }};

    File index = File.createTempFile("on-disk-sa-multi-suffix-match", ".db");
    index.deleteOnExit();

    builder.finish(index);

    OnDiskIndex onDisk = new OnDiskIndex(index, UTF8Type.instance, new KeyConverter());

    Assert.assertEquals(convert(1, 2, 3, 4, 5, 6), convert(onDisk.search(expressionFor("liz"))));
    Assert.assertEquals(convert(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), convert(onDisk.search(expressionFor("a"))));
    Assert.assertEquals(convert(5, 6), convert(onDisk.search(expressionFor("A"))));
    Assert.assertEquals(convert(1, 2, 3, 4), convert(onDisk.search(expressionFor("E"))));
    Assert.assertEquals(convert(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), convert(onDisk.search(expressionFor("l"))));
    Assert.assertEquals(convert(3, 4), convert(onDisk.search(expressionFor("bet"))));
    Assert.assertEquals(convert(3, 4, 9, 10), convert(onDisk.search(expressionFor("e"))));
    Assert.assertEquals(convert(7, 8), convert(onDisk.search(expressionFor("yl"))));
    Assert.assertEquals(convert(7, 8), convert(onDisk.search(expressionFor("T"))));
    Assert.assertEquals(convert(1, 2, 3, 4, 5, 6), convert(onDisk.search(expressionFor("za"))));
    Assert.assertEquals(convert(3, 4), convert(onDisk.search(expressionFor("ab"))));

    Assert.assertEquals(Collections.<DecoratedKey>emptySet(), convert(onDisk.search(expressionFor("Pi"))));
    Assert.assertEquals(Collections.<DecoratedKey>emptySet(), convert(onDisk.search(expressionFor("ethz"))));
    Assert.assertEquals(Collections.<DecoratedKey>emptySet(), convert(onDisk.search(expressionFor("liw"))));
    Assert.assertEquals(Collections.<DecoratedKey>emptySet(), convert(onDisk.search(expressionFor("Taw"))));
    Assert.assertEquals(Collections.<DecoratedKey>emptySet(), convert(onDisk.search(expressionFor("Av"))));

    onDisk.close();
}
 

private static void run(Descriptor desc, CommandLine cmd, PrintStream out) throws IOException {
  // Since we don't have a schema, make one up!
  CFMetaData cfm = new CFMetaData(desc.ksname, desc.cfname, ColumnFamilyType.Standard,
                                  UTF8Type.instance, UTF8Type.instance);

  SSTableReader reader = SSTableReader.open(desc, cfm);
  SSTableScanner scanner = reader.getScanner();

  long totalTombstones = 0, totalColumns = 0;
  if (cmd.hasOption("l")) {
    out.printf(desc.baseFilename() + "\n");
    out.printf("rowkey #tombstones (#columns)\n");
  }
  while (scanner.hasNext()) {
    SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();

    int tombstonesCount = 0, columnsCount = 0;
    while (row.hasNext())
    {
      OnDiskAtom column = row.next();
      long now = System.currentTimeMillis();
      if (column instanceof Column && ((Column) column).isMarkedForDelete(now)) {
        tombstonesCount++;
      }
      columnsCount++;
    }
    totalTombstones += tombstonesCount;
    totalColumns += columnsCount;

    if (tombstonesCount > 0) {
      String key;
      try {
        key = UTF8Type.instance.getString(row.getKey().key);
      } catch (RuntimeException e) {
        key = BytesType.instance.getString(row.getKey().key);
      }
      out.printf("%s %d (%d)%n", key, tombstonesCount, columnsCount);
    }

  }

  if (cmd.hasOption("l")) {
    out.printf("#total_tombstones (#total_columns)\n");
  }
  out.printf("%d (%d)%n", totalTombstones, totalColumns);

  scanner.close();
}
 

@Test
public void testRangeQueryWithExclusions() throws Exception
{
    final long lower = 0;
    final long upper = 100000;

    OnDiskIndexBuilder builder = new OnDiskIndexBuilder(UTF8Type.instance, LongType.instance, OnDiskIndexBuilder.Mode.SPARSE);
    for (long i = lower; i <= upper; i++)
        builder.add(LongType.instance.decompose(i), keyAt(i), i);

    File index = File.createTempFile("on-disk-sa-except-long-ranges", "db");
    index.deleteOnExit();

    builder.finish(index);

    OnDiskIndex onDisk = new OnDiskIndex(index, LongType.instance, new KeyConverter());

    ThreadLocalRandom random = ThreadLocalRandom.current();

    // single exclusion

    // let's do small range first to figure out if searchPoint works properly
    validateExclusions(onDisk, lower, 50, Sets.newHashSet(42L));
    // now let's do whole data set to test SPARSE searching
    validateExclusions(onDisk, lower, upper, Sets.newHashSet(31337L));

    // pair of exclusions which would generate a split

    validateExclusions(onDisk, lower, random.nextInt(400, 800), Sets.newHashSet(42L, 154L));
    validateExclusions(onDisk, lower, upper, Sets.newHashSet(31337L, 54631L));

    // 3 exclusions which would generate a split and change bounds

    validateExclusions(onDisk, lower, random.nextInt(400, 800), Sets.newHashSet(42L, 154L));
    validateExclusions(onDisk, lower, upper, Sets.newHashSet(31337L, 54631L));

    validateExclusions(onDisk, lower, random.nextLong(400, upper), Sets.newHashSet(42L, 55L));
    validateExclusions(onDisk, lower, random.nextLong(400, upper), Sets.newHashSet(42L, 55L, 93L));
    validateExclusions(onDisk, lower, random.nextLong(400, upper), Sets.newHashSet(42L, 55L, 93L, 205L));

    Set<Long> exclusions = Sets.newHashSet(3L, 12L, 13L, 14L, 27L, 54L, 81L, 125L, 384L, 771L, 1054L, 2048L, 78834L);

    // test that exclusions are properly bound by lower/upper of the expression
    Assert.assertEquals(392, validateExclusions(onDisk, lower, 400, exclusions, false));
    Assert.assertEquals(101, validateExclusions(onDisk, lower, 100, Sets.newHashSet(-10L, -5L, -1L), false));

    validateExclusions(onDisk, lower, upper, exclusions);

    Assert.assertEquals(100000, convert(onDisk.search(new Expression(ByteBufferUtil.EMPTY_BYTE_BUFFER, LongType.instance)
                                                .add(IndexOperator.NOT_EQ, LongType.instance.decompose(100L)))).size());

    Assert.assertEquals(49, convert(onDisk.search(new Expression(ByteBufferUtil.EMPTY_BYTE_BUFFER, LongType.instance)
                                                .add(IndexOperator.LT, LongType.instance.decompose(50L))
                                                .add(IndexOperator.NOT_EQ, LongType.instance.decompose(10L)))).size());

    Assert.assertEquals(99998, convert(onDisk.search(new Expression(ByteBufferUtil.EMPTY_BYTE_BUFFER, LongType.instance)
                                                .add(IndexOperator.GT, LongType.instance.decompose(1L))
                                                .add(IndexOperator.NOT_EQ, LongType.instance.decompose(20L)))).size());

    onDisk.close();
}
 

public AbstractType<?> getTokenValidator()
{
    return UTF8Type.instance;
}
 

/**
 * Builds a new {@link ColumnMapperBlob}.
 */
@JsonCreator
public ColumnMapperBlob() {
    super(new AbstractType<?>[]{AsciiType.instance, UTF8Type.instance, BytesType.instance}, new AbstractType[]{});
}
 

/**
 * Builds a new {@link ColumnMapperInet}.
 */
@JsonCreator
public ColumnMapperInet() {
    super(new AbstractType<?>[]{AsciiType.instance, UTF8Type.instance, InetAddressType.instance},
          new AbstractType[]{});
}
 

/**
 * Builds a new {@link ColumnMapperBlob}.
 */
@JsonCreator
public ColumnMapperBoolean() {
    super(new AbstractType<?>[]{AsciiType.instance, UTF8Type.instance, BooleanType.instance}, new AbstractType[]{});
}
 

public HexStrings(String name, GeneratorConfig config)
{
    super(UTF8Type.instance, config, name, String.class);
    chars = new char[(int) sizeDistribution.maxValue()];
}
 

@Test
public void testNotEqualsQueryForStrings() throws Exception
{
    Map<ByteBuffer, TokenTreeBuilder> data = new HashMap<ByteBuffer, TokenTreeBuilder>()
    {{
            put(UTF8Type.instance.decompose("Pavel"),   keyBuilder(1L, 2L));
            put(UTF8Type.instance.decompose("Jason"),   keyBuilder(3L));
            put(UTF8Type.instance.decompose("Jordan"),  keyBuilder(4L));
            put(UTF8Type.instance.decompose("Michael"), keyBuilder(5L, 6L));
            put(UTF8Type.instance.decompose("Vijay"),   keyBuilder(7L));
            put(UTF8Type.instance.decompose("Travis"),  keyBuilder(8L));
            put(UTF8Type.instance.decompose("Aleksey"), keyBuilder(9L, 10L));
    }};

    OnDiskIndexBuilder builder = new OnDiskIndexBuilder(UTF8Type.instance, UTF8Type.instance, OnDiskIndexBuilder.Mode.ORIGINAL);
    for (Map.Entry<ByteBuffer, TokenTreeBuilder> e : data.entrySet())
        addAll(builder, e.getKey(), e.getValue());

    File index = File.createTempFile("on-disk-sa-except-test", "db");
    index.deleteOnExit();

    builder.finish(index);

    OnDiskIndex onDisk = new OnDiskIndex(index, UTF8Type.instance, new KeyConverter());

    // test whole words first
    Assert.assertEquals(convert(3, 4, 5, 6, 7, 8, 9, 10), convert(onDisk.search(expressionForNot("Aleksey", "Vijay", "Pavel"))));

    Assert.assertEquals(convert(3, 4, 7, 8, 9, 10), convert(onDisk.search(expressionForNot("Aleksey", "Vijay", "Pavel", "Michael"))));

    Assert.assertEquals(convert(3, 4, 7, 9, 10), convert(onDisk.search(expressionForNot("Aleksey", "Vijay", "Pavel", "Michael", "Travis"))));

    // now test prefixes
    Assert.assertEquals(convert(3, 4, 5, 6, 7, 8, 9, 10), convert(onDisk.search(expressionForNot("Aleksey", "Vijay", "Pav"))));

    Assert.assertEquals(convert(3, 4, 7, 8, 9, 10), convert(onDisk.search(expressionForNot("Aleksey", "Vijay", "Pavel", "Mic"))));

    Assert.assertEquals(convert(3, 4, 7, 9, 10), convert(onDisk.search(expressionForNot("Aleksey", "Vijay", "Pavel", "Micha", "Tr"))));

    onDisk.close();
}
 

@VisibleForTesting
public Expression(ByteBuffer name, AbstractType<?> validator)
{
    this(null, new ColumnIndex(UTF8Type.instance, ColumnDefinition.regularDef(name, validator, 0), UTF8Type.instance));
}