Java Code Examples for com.indeed.util.serialization.Serializer#write()

private static<K,V> boolean updateBuffer(
        final Generation.Entry<K,V> entry, Serializer<K> keySerializer, Serializer<V> valueSerializer, final boolean keepDeletions, final DataOutput bufferDataOutput
) throws IOException {
    final boolean skipDeleted;
    if (keepDeletions) {
        skipDeleted = false;
        keySerializer.write(entry.getKey(), bufferDataOutput);
        if (entry.isDeleted()) {
            bufferDataOutput.writeByte(1);
        } else {
            bufferDataOutput.writeByte(0);
            valueSerializer.write(entry.getValue(), bufferDataOutput);
        }
    } else {
        if (entry.isDeleted()) {
            skipDeleted = true;
        } else {
            skipDeleted = false;
            keySerializer.write(entry.getKey(), bufferDataOutput);
            valueSerializer.write(entry.getValue(), bufferDataOutput);
        }
    }
    return skipDeleted;
}
 

MemcachedCache(
        final MemcachedClient memcache,
        final InetSocketAddress address,
        final String prefix,
        final Stringifier<K> keyStringifier,
        final Serializer<V> valueSerializer
) throws IOException {
    this.memcache = memcache;
    this.prefix = prefix;
    this.keyStringifier = keyStringifier;
    this.host = address;
    valueTranscoder = new Transcoder<V>() {
        @Override
        public boolean asyncDecode(CachedData cachedData) {
            return false;
        }

        @Override
        public CachedData encode(V v) {
            ByteArrayDataOutput out = ByteStreams.newDataOutput();
            try {
                valueSerializer.write(v, out);
                return identityTranscoder.encode(out.toByteArray());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }

        @Override
        public V decode(CachedData cachedData) {
            byte[] bytes = identityTranscoder.decode(cachedData);
            ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
            try {
                return valueSerializer.read(in);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }

        @Override
        public int getMaxSize() {
            return Integer.MAX_VALUE;
        }
    };
    final Thread addFutureChecker = new Thread(
            new FutureQueueChecker(run, addFutureQueue, "memcached add failed, key already exists in cache", Level.INFO),
            "addFutureChecker"
    );
    addFutureChecker.setDaemon(true);
    addFutureChecker.start();
    final Thread setFutureChecker = new Thread(
            new FutureQueueChecker(run, setFutureQueue, "memcached set failed, this should never happen", Level.ERROR),
            "setFutureChecker"
    );
    setFutureChecker.setDaemon(true);
    setFutureChecker.start();
}
 

private static <K,V> WriteLevelResult writeLevel(
        final CountingOutputStream counter,
        final Path tempPath,
        final Iterator<Generation.Entry<K,V>> iterator,
        final Serializer<K> keySerializer,
        final Serializer<V> valueSerializer,
        final int blocksize,
        final boolean keepDeletions
) throws IOException {
    Generation.Entry<K,V> next;
    if (!iterator.hasNext()) {
        return new WriteLevelResult(0, 0);
    }
    next = iterator.next();
    final LittleEndianDataOutputStream tmpOut = new LittleEndianDataOutputStream(new BufferedOutputStream(Files.newOutputStream(tempPath), 131072));
    final ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    final LittleEndianDataOutputStream bufferDataOutput = new LittleEndianDataOutputStream(buffer);
    final ByteArrayOutputStream currentBlock = new ByteArrayOutputStream(blocksize);
    final CharArrayList keyOffsets = new CharArrayList();
    int tmpCount = 0;
    boolean done = false;
    final LittleEndianDataOutputStream out = new LittleEndianDataOutputStream(counter);
    long count = 0;
    outer: while (!done) {

        currentBlock.reset();
        keyOffsets.clear();
        if (!keepDeletions) {
            while (next.isDeleted()) {
                if (!iterator.hasNext()) break outer;
                next = iterator.next();
            }
        }
        keySerializer.write(next.getKey(), tmpOut);
        tmpOut.writeLong(counter.getCount());
        tmpCount++;
        while (true) {
            buffer.reset();
            final boolean skipDeleted = updateBuffer(next, keySerializer, valueSerializer, keepDeletions, bufferDataOutput);
            if (4+2*keyOffsets.size()+2+currentBlock.size()+buffer.size() > blocksize) {
                if (currentBlock.size() == 0) {
                    throw new IllegalArgumentException("key value pair is greater than block size");
                }
                break;
            }
            if (!skipDeleted) {
                keyOffsets.add((char)currentBlock.size());
                buffer.writeTo(currentBlock);
                count++;
            }
            if (!iterator.hasNext()) {
                done = true;
                break;
            }
            next = iterator.next();
        }
        if (keyOffsets.size() > 0) {
            final long start = counter.getCount();
            out.writeInt(keyOffsets.size());
            for (int i = 0; i < keyOffsets.size(); i++) {
                out.writeChar(keyOffsets.getChar(i));
            }
            currentBlock.writeTo(out);
            if (counter.getCount()-start > blocksize) {
                log.error("too big");
            }
        }
    }
    tmpOut.close();
    return new WriteLevelResult(tmpCount, count);
}
 

public static <K> void write(MemoryManager memoryManager, File file, Iterator<? extends Generation.Entry<K, ?>> iterator, Serializer<K> keySerializer, long size)
        throws IOException, NotEnoughMemoryException {
    final MemoryManager.AddressSpace addressSpace = memoryManager.open(file, size);
    final Memory[] pages = new Memory[(int)((size-1) >> PAGE_BITS)+1];
    long pageIndex = 0;
    try {
        for (pageIndex = 0; pageIndex < size; pageIndex += PAGE_SIZE) {
            final Memory page = addressSpace.getPageForWriting(pageIndex);
            if (page == null) {
                throw new NotEnoughMemoryException("insufficient memory available to write bloom filter, allocated "+pageIndex+" bytes before failing");
            }
            pages[((int)(pageIndex >>> PAGE_BITS))] = page;
        }
    } catch (Throwable t) {
        for (long j = 0; j < pageIndex; j += PAGE_SIZE) {
            addressSpace.releasePage(j);
        }
        addressSpace.close();
        Throwables.propagateIfInstanceOf(t, IOException.class);
        Throwables.propagateIfInstanceOf(t, NotEnoughMemoryException.class);
        throw Throwables.propagate(t);
    }
    try {
        final ByteArrayOutputStream out = new ByteArrayOutputStream();
        final DataOutputStream dataOut = new DataOutputStream(out);
        while (iterator.hasNext()) {
            final Generation.Entry<K, ?> next = iterator.next();
            final K key = next.getKey();
            out.reset();
            keySerializer.write(key, dataOut);
            final byte[] bytes = out.toByteArray();
            int previousHash = MurmurHash.SEED64;
            for (int i = 0; i < NUM_HASHES; i++) {
                long hash = getHash(bytes, previousHash);
                previousHash = (int)hash;
                hash = hash % (size*8);
                final long byteIndex = hash>>>3;
                final int bitIndex = (int)(hash&7);
                final Memory memory = pages[((int)(byteIndex >>> PAGE_BITS))];
                int current = memory.getByte(byteIndex & PAGE_OFFSET_MASK)&0xFF;
                current |= 1<<bitIndex;
                memory.putByte(byteIndex & PAGE_OFFSET_MASK, (byte)current);
            }
        }
    } finally {
        for (long i = 0; i < size; i += PAGE_SIZE) {
            addressSpace.releasePage(i);
        }
        addressSpace.close();
    }
}