com.carrotsearch.hppc.LongSet Java Examples

private LeafEntry createEntry(final long tok, final LongSet offsets)
{
    int offsetCount = offsets.size();
    switch (offsetCount)
    {
        case 0:
            throw new AssertionError("no offsets for token " + tok);
        case 1:
            long offset = offsets.toArray()[0];
            if (offset > MAX_OFFSET)
                throw new AssertionError("offset " + offset + " cannot be greater than " + MAX_OFFSET);
            else if (offset <= Integer.MAX_VALUE)
                return new SimpleLeafEntry(tok, offset);
            else
                return new FactoredOffsetLeafEntry(tok, offset);
        case 2:
            long[] rawOffsets = offsets.toArray();
            if (rawOffsets[0] <= Integer.MAX_VALUE && rawOffsets[1] <= Integer.MAX_VALUE &&
                    (rawOffsets[0] <= Short.MAX_VALUE || rawOffsets[1] <= Short.MAX_VALUE))
                return new PackedCollisionLeafEntry(tok, rawOffsets);
            else
                return createOverflowEntry(tok, offsetCount, offsets);
        default:
            return createOverflowEntry(tok, offsetCount, offsets);
    }
}
 

@Override
protected boolean matchesSafely(Translog.Snapshot snapshot) {
    try {
        final LongSet seqNoList = new LongHashSet();
        Translog.Operation op;
        while ((op = snapshot.next()) != null) {
            seqNoList.add(op.seqNo());
        }
        for (long i = minSeqNo; i <= maxSeqNo; i++) {
            if (seqNoList.contains(i) == false) {
                notFoundSeqNo.add(i);
            }
        }
        return notFoundSeqNo.isEmpty();
    } catch (IOException ex) {
        throw new ElasticsearchException("failed to read snapshot content", ex);
    }
}
 

@Test
public void buildAndIterate() throws Exception
{
    final TokenTreeBuilder builder = new TokenTreeBuilder(tokens).finish();
    final Iterator<Pair<Long, LongSet>> tokenIterator = builder.iterator();
    final Iterator<Map.Entry<Long, LongSet>> listIterator = tokens.entrySet().iterator();
    while (tokenIterator.hasNext() && listIterator.hasNext())
    {
        Pair<Long, LongSet> tokenNext = tokenIterator.next();
        Map.Entry<Long, LongSet> listNext = listIterator.next();

        Assert.assertEquals(listNext.getKey(), tokenNext.left);
        Assert.assertEquals(listNext.getValue(), tokenNext.right);
    }

    Assert.assertFalse("token iterator not finished", tokenIterator.hasNext());
    Assert.assertFalse("list iterator not finished", listIterator.hasNext());
}
 

@Override
public Pair<Long, LongSet> computeNext()
{
    if (currentIterator != null && currentIterator.hasNext())
    {
        Map.Entry<Long, LongSet> next = currentIterator.next();
        return Pair.create(next.getKey(), next.getValue());
    }
    else
    {
        if (!levelIterator.hasNext())
            return endOfData();
        else
        {
            currentIterator = ((Leaf) levelIterator.next()).tokenIterator();
            return computeNext();
        }
    }

}
 

public CombinedTerm(AbstractType<?> comparator, DataTerm term)
{
    this.comparator = comparator;
    this.term = term;
    this.tokens = new TreeMap<>();

    RangeIterator<Long, Token> tokens = term.getTokens();
    while (tokens.hasNext())
    {
        Token current = tokens.next();
        LongSet offsets = this.tokens.get(current.get());
        if (offsets == null)
            this.tokens.put(current.get(), (offsets = new LongOpenHashSet()));

        for (Long offset : ((TokenTree.OnDiskToken) current).getOffsets())
            offsets.add(offset);
    }
}
 

@Test
public void testSimpleIDAcquisition() throws BackendException {
    final IDBlockSizer blockSizer = new InnerIDBlockSizer();
    idAuthorities[0].setIDBlockSizer(blockSizer);
    int numTrials = 100;
    LongSet ids = new LongHashSet((int)blockSize*numTrials);
    long previous = 0;
    for (int i=0;i<numTrials;i++) {
        IDBlock block = idAuthorities[0].getIDBlock(0, 0, GET_ID_BLOCK_TIMEOUT);
        checkBlock(block,ids);
        if (hasEmptyUid) {
            if (previous!=0)
                assertEquals(previous+1, block.getId(0));
            previous=block.getId(block.numIds()-1);
        }
    }
}
 

private void checkBlock(IDBlock block, LongSet ids) {
    assertEquals(blockSize,block.numIds());
    for (int i=0;i<blockSize;i++) {
        long id = block.getId(i);
        assertEquals(id,block.getId(i));
        assertFalse(ids.contains(id));
        assertTrue(id<idUpperBound);
        assertTrue(id>0);
        ids.add(id);
    }
    if (hasEmptyUid) {
        assertEquals(blockSize-1,block.getId(block.numIds()-1)-block.getId(0));
    }
    try {
        block.getId(blockSize);
        fail();
    } catch (ArrayIndexOutOfBoundsException e) {}
}
 

private static TokenTree generateTree(final long minToken, final long maxToken) throws IOException
{
    final SortedMap<Long, LongSet> toks = new TreeMap<Long, LongSet>()
    {{
            for (long i = minToken; i <= maxToken; i++)
            {
                LongSet offsetSet = new LongOpenHashSet();
                offsetSet.add(i);
                put(i, offsetSet);
            }
    }};

    final TokenTreeBuilder builder = new TokenTreeBuilder(toks).finish();
    final File treeFile = File.createTempFile("token-tree-get-test", "tt");
    treeFile.deleteOnExit();

    final SequentialWriter writer = new SequentialWriter(treeFile, 4096, false);
    builder.write(writer);
    writer.close();

    RandomAccessReader reader = null;

    try
    {
        reader = RandomAccessReader.open(treeFile);
        return new TokenTree(new MappedBuffer(reader));
    }
    finally
    {
        FileUtils.closeQuietly(reader);
    }
}
 

public void add(Long token, long keyPosition)
{
    LongSet found = tokens.get(token);
    if (found == null)
        tokens.put(token, (found = new LongOpenHashSet(2)));

    found.add(keyPosition);
}
 

public void add(SortedMap<Long, LongSet> data)
{
    for (Map.Entry<Long, LongSet> newEntry : data.entrySet())
    {
        LongSet found = tokens.get(newEntry.getKey());
        if (found == null)
            tokens.put(newEntry.getKey(), (found = new LongOpenHashSet(4)));

        for (LongCursor offset : newEntry.getValue())
            found.add(offset.value);
    }
}
 

private static LongSet convert(long... values)
{
    LongSet result = new LongOpenHashSet(values.length);
    for (long v : values)
        result.add(v);

    return result;
}
 

private static Set<DecoratedKey> convert(LongSet offsets)
{
    Set<DecoratedKey> keys = new HashSet<>();
    for (LongCursor offset : offsets)
        keys.add(KEY_CONVERTER.apply(offset.value));

    return keys;
}
 

@Override
public TokenWithOffsets computeNext()
{
    if (!elements.hasNext())
        return endOfData();

    Map.Entry<Long, LongSet> next = elements.next();
    return new TokenWithOffsets(next.getKey(), next.getValue());
}
 

@Test
public void buildSerializeAndIterate() throws Exception
{
    final TokenTreeBuilder builder = new TokenTreeBuilder(simpleTokenMap).finish();

    final File treeFile = File.createTempFile("token-tree-iterate-test1", "tt");
    treeFile.deleteOnExit();

    final SequentialWriter writer = new SequentialWriter(treeFile, 4096, false);
    builder.write(writer);
    writer.close();

    final RandomAccessReader reader = RandomAccessReader.open(treeFile);
    final TokenTree tokenTree = new TokenTree(new MappedBuffer(reader));

    final Iterator<Token> tokenIterator = tokenTree.iterator(KEY_CONVERTER);
    final Iterator<Map.Entry<Long, LongSet>> listIterator = simpleTokenMap.entrySet().iterator();
    while (tokenIterator.hasNext() && listIterator.hasNext())
    {
        Token treeNext = tokenIterator.next();
        Map.Entry<Long, LongSet> listNext = listIterator.next();

        Assert.assertEquals(listNext.getKey(), treeNext.get());
        Assert.assertEquals(convert(listNext.getValue()), convert(treeNext));
    }

    Assert.assertFalse("token iterator not finished", tokenIterator.hasNext());
    Assert.assertFalse("list iterator not finished", listIterator.hasNext());

    reader.close();
}
 

private void writeTerm(InMemoryTerm term, TokenTreeBuilder keys) throws IOException
{
    term.serialize(buffer);
    buffer.writeByte((byte) keys.getTokenCount());

    Iterator<Pair<Long, LongSet>> tokens = keys.iterator();
    while (tokens.hasNext())
        buffer.writeLong(tokens.next().left);
}
 

private static Set<DecoratedKey> convert(TokenTreeBuilder offsets)
{
    Set<DecoratedKey> result = new HashSet<>();

    Iterator<Pair<Long, LongSet>> offsetIter = offsets.iterator();
    while (offsetIter.hasNext())
    {
        LongSet v = offsetIter.next().right;

        for (LongCursor offset : v)
            result.add(keyAt(offset.value));
    }
    return result;
}
 

private static void addAll(OnDiskIndexBuilder builder, ByteBuffer term, TokenTreeBuilder tokens)
{
    for (Map.Entry<Long, LongSet> token : tokens.getTokens().entrySet())
    {
        for (long position : token.getValue().toArray())
            builder.add(term, keyAt(position), position);
    }
}
 

private void checkBlock(IDBlock block) {
    assertTrue(blockSize<10000);
    LongSet ids = new LongHashSet((int)blockSize);
    checkBlock(block,ids);
}
 

@Test
public void testMultiIDAcquisition() throws Throwable {
    final int numPartitions = MAX_NUM_PARTITIONS;
    final int numAcquisitionsPerThreadPartition = 100;
    final IDBlockSizer blockSizer = new InnerIDBlockSizer();
    for (int i = 0; i < CONCURRENCY; i++) idAuthorities[i].setIDBlockSizer(blockSizer);
    final List<ConcurrentLinkedQueue<IDBlock>> ids = new ArrayList<ConcurrentLinkedQueue<IDBlock>>(numPartitions);
    for (int i = 0; i < numPartitions; i++) {
        ids.add(new ConcurrentLinkedQueue<IDBlock>());
    }

    final int maxIterations = numAcquisitionsPerThreadPartition * numPartitions * 2;
    final Collection<Future<?>> futures = new ArrayList<Future<?>>(CONCURRENCY);
    ExecutorService es = Executors.newFixedThreadPool(CONCURRENCY);

    Set<String> uids = new HashSet<String>(CONCURRENCY);
    for (int i = 0; i < CONCURRENCY; i++) {
        final IDAuthority idAuthority = idAuthorities[i];
        final IDStressor stressRunnable = new IDStressor(
                numAcquisitionsPerThreadPartition, numPartitions,
                maxIterations, idAuthority, ids);
        uids.add(idAuthority.getUniqueID());
        futures.add(es.submit(stressRunnable));
    }

    // If this fails, it's likely to be a bug in the test rather than the
    // IDAuthority (the latter is technically possible, just less likely)
    assertEquals(CONCURRENCY, uids.size());

    for (Future<?> f : futures) {
        try {
            f.get();
        } catch (ExecutionException e) {
            throw e.getCause();
        }
    }

    for (int i = 0; i < numPartitions; i++) {
        ConcurrentLinkedQueue<IDBlock> list = ids.get(i);
        assertEquals(numAcquisitionsPerThreadPartition * CONCURRENCY, list.size());
        LongSet idset = new LongHashSet((int)blockSize*list.size());
        for (IDBlock block : list) checkBlock(block,idset);
    }

    es.shutdownNow();
}
 

public TokenWithOffsets(long token, final LongSet offsets)
{
    super(token);
    this.offsets = offsets;
}
 

EntrySetSkippableIterator(SortedMap<Long, LongSet> elms)
{
    super(elms.firstKey(), elms.lastKey(), elms.size());
    elements = Iterators.peekingIterator(elms.entrySet().iterator());
}
 

@Test
public void buildWithMultipleMapsAndIterate() throws Exception
{
    final SortedMap<Long, LongSet> merged = new TreeMap<>();
    final TokenTreeBuilder builder = new TokenTreeBuilder(simpleTokenMap).finish();
    builder.add(collidingTokensMap);

    merged.putAll(collidingTokensMap);
    for (Map.Entry<Long, LongSet> entry : simpleTokenMap.entrySet())
    {
        if (merged.containsKey(entry.getKey()))
        {
            LongSet mergingOffsets  = entry.getValue();
            LongSet existingOffsets = merged.get(entry.getKey());

            if (mergingOffsets.equals(existingOffsets))
                continue;

            Set<Long> mergeSet = new HashSet<>();
            for (LongCursor merging : mergingOffsets)
                mergeSet.add(merging.value);

            for (LongCursor existing : existingOffsets)
                mergeSet.add(existing.value);

            LongSet mergedResults = new LongOpenHashSet();
            for (Long result : mergeSet)
                mergedResults.add(result);

            merged.put(entry.getKey(), mergedResults);
        }
        else
        {
            merged.put(entry.getKey(), entry.getValue());
        }
    }

    final Iterator<Pair<Long, LongSet>> tokenIterator = builder.iterator();
    final Iterator<Map.Entry<Long, LongSet>> listIterator = merged.entrySet().iterator();
    while (tokenIterator.hasNext() && listIterator.hasNext())
    {
        Pair<Long, LongSet> tokenNext = tokenIterator.next();
        Map.Entry<Long, LongSet> listNext = listIterator.next();

        Assert.assertEquals(listNext.getKey(), tokenNext.left);
        Assert.assertEquals(listNext.getValue(), tokenNext.right);
    }

    Assert.assertFalse("token iterator not finished", tokenIterator.hasNext());
    Assert.assertFalse("list iterator not finished", listIterator.hasNext());

}
 

public InSet(Expression child, LongSet hset) {
    this.child = child;
    this.hset = hset;
    assert !hset.isEmpty();
}
 

private void serializeData(ByteBuffer buf)
{
    for (Map.Entry<Long, LongSet> entry : tokens.entrySet())
        createEntry(entry.getKey(), entry.getValue()).serialize(buf);
}
 

public Iterator<Map.Entry<Long, LongSet>> tokenIterator()
{
    return tokens.entrySet().iterator();
}
 

Leaf(SortedMap<Long, LongSet> data)
{
    nodeMinToken = data.firstKey();
    nodeMaxToken = data.lastKey();
    tokens = data;
}
 

public Iterator<Pair<Long, LongSet>> iterator()
{
    return new TokenIterator(leftmostLeaf.levelIterator());
}
 

public SortedMap<Long, LongSet> getTokens()
{
    return tokens;
}
 

public TokenTreeBuilder(SortedMap<Long, LongSet> data)
{
    add(data);
}
 

public Map<Long, LongSet> getTokens()
{
    return tokens;
}