com.indeed.util.serialization.Serializer Java Examples

private static<K,V> boolean updateBuffer(
        final Generation.Entry<K,V> entry, Serializer<K> keySerializer, Serializer<V> valueSerializer, final boolean keepDeletions, final DataOutput bufferDataOutput
) throws IOException {
    final boolean skipDeleted;
    if (keepDeletions) {
        skipDeleted = false;
        keySerializer.write(entry.getKey(), bufferDataOutput);
        if (entry.isDeleted()) {
            bufferDataOutput.writeByte(1);
        } else {
            bufferDataOutput.writeByte(0);
            valueSerializer.write(entry.getValue(), bufferDataOutput);
        }
    } else {
        if (entry.isDeleted()) {
            skipDeleted = true;
        } else {
            skipDeleted = false;
            keySerializer.write(entry.getKey(), bufferDataOutput);
            valueSerializer.write(entry.getValue(), bufferDataOutput);
        }
    }
    return skipDeleted;
}
 

public BlockCompressedRecordFile(final Supplier<? extends Either<IOException, ? extends RandomAccessDataInput>> inputSupplier, final Closeable closeable, String file, Serializer<E> serializer, CompressionCodec codec, BlockingQueue<Decompressor> decompressorPool, int blockSize, int recordIndexBits, int padBits, int maxChunkSize) throws IOException {
    this.inputSupplier = inputSupplier;
    this.file = file;
    this.serializer = serializer;
    this.codec = codec;
    this.blockSize = blockSize;
    this.padBits = padBits;
    this.maxChunkSize = maxChunkSize;
    pad = 1<<padBits;
    padMask = ~(long)(pad-1);
    shift = Math.max(recordIndexBits - padBits, 0);
    mask = (1L<<recordIndexBits)-1;
    closeableRef = SharedReference.create(closeable);
    try {
        blockCache = new BlockCache(decompressorPool);
    } catch (Throwable t) {
        Closeables2.closeQuietly(closeableRef, log);
        Throwables.propagateIfInstanceOf(t, IOException.class);
        throw Throwables.propagate(t);
    }
}
 

public BlockCompressedStableGeneration(
        BloomFilter.MemoryManager memoryManager, File file, Comparator<K> comparator, Serializer<K> keySerializer, Serializer<V> valueSerializer, final CompressionCodec codec, final boolean mlockBTree
) throws IOException {
    this.file = file;
    reader = new ImmutableBTreeIndex.Reader(file, comparator, keySerializer, new LongSerializer(), mlockBTree);
    final File valuesFile = new File(file, "values.bin");
    recordFile =
            new BlockCompressedRecordFile.Builder(valuesFile, valueSerializer, codec).setMlockFiles(mlockBTree).build();
    final File bloomFilterFile = new File(file, "bloomfilter.bin");
    if (bloomFilterFile.exists()) {
        bloomFilter = new BloomFilter.Reader(memoryManager, bloomFilterFile, keySerializer);
    } else {
        bloomFilter = null;
    }
    sizeInBytes = reader.sizeInBytes()+valuesFile.length()+(bloomFilter == null ? 0 : bloomFilter.sizeInBytes());
    stuffToClose = SharedReference.create((Closeable)new Closeable() {
        public void close() throws IOException {
            Closeables2.closeQuietly(reader, log);
            if (bloomFilter != null) Closeables2.closeQuietly(bloomFilter, log);
            Closeables2.closeQuietly(recordFile, log);
        }
    });
}
 

public VolatileGeneration(File logPath, Serializer<K> keySerializer, Serializer<V> valueSerializer, Comparator<K> comparator, boolean loadExistingReadOnly) throws IOException {
    this.ordering = Ordering.from(comparator);
    map = new ConcurrentSkipListMap(comparator);
    this.logPath = logPath;
    this.keySerializer = keySerializer;
    this.valueSerializer = valueSerializer;
    deleted = new Object();
    if (loadExistingReadOnly) {
        if (!logPath.exists()) throw new IllegalArgumentException(logPath.getAbsolutePath()+" does not exist");
        transactionLog = null;
        replayTransactionLog(logPath, true);
    } else {
        if (logPath.exists()) throw new IllegalArgumentException("to load existing logs set loadExistingReadOnly to true or create a new log and use replayTransactionLog");
        transactionLog = new TransactionLog.Writer(logPath, keySerializer, valueSerializer, false);
    }
    stuffToClose = SharedReference.create((Closeable)new Closeable() {
        public void close() throws IOException {
            closeWriter();
        }
    });
}
 

public RecordLogDirectory(
        File dir,
        Serializer<E> serializer,
        int maxCachedFiles,
        CompressionCodec codec,
        int blockSize,
        int fileIndexBits,
        int recordIndexBits,
        int padBits,
        boolean mlockFiles
) {
    this.dir = dir;
    this.serializer = serializer;
    this.maxCachedFiles = maxCachedFiles;
    this.codec = codec;
    this.blockSize = blockSize;
    this.fileIndexBits = fileIndexBits;
    this.recordIndexBits = recordIndexBits;
    this.padBits = padBits;
    segmentShift = 64-fileIndexBits;
    segmentMask = (1L<< segmentShift)-1;
    fileCache = new FileCache(mlockFiles);
}
 

private Writer(
        File file, Serializer<E> serializer, CompressionCodec codec, final long rollFrequency, int blockSize, int fileIndexBits, int recordIndexBits, int padBits, int maxSegment
) throws IOException {
    this.path = file;
    this.serializer = serializer;
    this.codec = codec;
    this.blockSize = blockSize;
    this.recordIndexBits = recordIndexBits;
    this.padBits = padBits;
    segmentShift = 64-fileIndexBits;
    this.tmpPath = new File(file, "tmp");
    this.rollFrequency = rollFrequency;
    tmpPath.mkdirs();
    if (maxSegment < 0) {
        currentSegmentNum = getMaxSegmentNum(path);
        if (currentSegmentNum == -1 || verifySegmentIntegrity(path, currentSegmentNum)) currentSegmentNum++;
    } else {
        currentSegmentNum = maxSegment+1;
    }
    log.info("current segment num: "+currentSegmentNum);
    currentWriter = createWriter(currentSegmentNum);
    lastRollTime = System.currentTimeMillis();
}
 

public static void main(final String[] args) throws IOException {
    SmartSerializer keySerializer = new SmartStringSerializer();
    SmartSerializer valueSerializer = new SmartStringSerializer();
    int i = 0;
    parse_opts:
    for ( ; i < args.length && args[i].startsWith("-"); ++i) {
        switch (args[i]) {
        case "--":
            break parse_opts;
        case "--keySerializer":
            keySerializer = TableWriter.parseSerializer(args[++i]); break;
        case "--valueSerializer":
            valueSerializer = TableWriter.parseSerializer(args[++i]); break;
        default:
            throw new RuntimeException("unknown option: " + args[i]);
        }
    }
    if (args.length - i < 2) {
        throw new RuntimeException("usage: WriteLsmTree [options] <dir> <inputs> ...");
    }
    final long startTime = System.currentTimeMillis();
    final File outputDir = new File(args[i]);
    final List<File> files = new ArrayList<>();
    for (int j = i + 1; j < args.length; ++j) {
        files.add(new File(args[j]));
    }
    final Store<Object, Object> store =
        new StoreBuilder<Object, Object>(outputDir, (Serializer<Object>) keySerializer, (Serializer<Object>) valueSerializer)
           .build();
    for (final Pair<Object, Object> p : new TableWriter.TsvFileReader<Object, Object>(files, (Parseable<Object>) keySerializer, (Parseable<Object>) valueSerializer, "\t", null, null, 0.0)) {
        store.put(p.getFirst(), p.getSecond());
    }
    store.close();
    System.out.println("completed in " + (System.currentTimeMillis() - startTime) + " ms");
}
 

/**
 * @param file              root directory for record logs
 * @param serializer        serializer
 * @param codec             compression codec
 * @param metadataRef       if non null, this will contain a reference to metadata after construction if it existed
 * @param rollFrequency     how frequently new record files should be created in milliseconds, but only if there
 *                          is a new write. If set to {@link Long#MAX_VALUE}, new record files will only be created when
 *                          {@link #flushWriter(java.util.Map)} is called
 *
 * @throws IOException      if an I/O error occurs
 */
public GenericRecordLogAppender(@Nonnull File file,
                                @Nonnull Serializer<T> serializer,
                                @Nonnull CompressionCodec codec,
                                @Nullable AtomicReference<Map<String, String>> metadataRef,
                                long rollFrequency) throws IOException {
    mapper = new ObjectMapper();
    mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
    this.file = file;
    lastPositionPath = new File(this.file, "lastposition.txt");
    maxSegmentPath = new File(file, "maxsegment.txt");
    metadataPath = new File(file, "metadata.json");
    if (metadataPath.exists()) {
        Map<String, String> metadata = readMetadata(metadataPath, mapper);
        if (metadataRef != null) metadataRef.set(metadata);
        lastPosition = Long.parseLong(metadata.get(LAST_POSITION_KEY));
        maxSegment = Integer.parseInt(metadata.get(MAX_SEGMENT_KEY));
        log.info("lastposition: "+lastPosition);
        log.info("maxsegment: "+maxSegment);
    } else {
        lastPosition = readLongFromFile(lastPositionPath, 0);
        maxSegment = -1;
    }
    writer = maxSegment < 0 ?
            RecordLogDirectory.Writer.create(file, serializer, codec, rollFrequency) :
            RecordLogDirectory.Writer.create(file, serializer, codec, rollFrequency, maxSegment);
}
 

/**
 * @param path root lsm tree index directory
 * @param iterator the iterator
 * @param keySerializer the key serializer
 * @param valueSerializer the value serializer
 * @param blocksize block size
 * @param keepDeletions true to keep deletion
 * @param <K> the key type
 * @param <V> the value type
 * @throws IOException  if an I/O error occurs
 */
public static <K, V> void write(
        Path path,
        Iterator<Generation.Entry<K,V>> iterator,
        Serializer<K> keySerializer,
        Serializer<V> valueSerializer,
        final int blocksize,
        boolean keepDeletions
) throws IOException {
    if (blocksize > 65536) throw new IllegalArgumentException("block size must be less than 65536");
    Files.createDirectories(path);
    final BufferedFileDataOutputStream fileOut = new BufferedFileDataOutputStream(path.resolve("index.bin"));
    final CountingOutputStream out = new CountingOutputStream(fileOut);
    //tempFile is deleted in writeIndex
    final Path tempPath = Files.createTempFile("tmp", ".bin");
    final WriteLevelResult result = writeLevel(out, tempPath, iterator, keySerializer, valueSerializer, blocksize, keepDeletions);
    final int tmpCount = result.tmpCount;
    final long size = result.size;

    final long valueLevelLength = out.getCount();
    final Header header = writeIndex(out, tempPath, tmpCount, keySerializer, blocksize);
    header.valueLevelLength = valueLevelLength;
    header.size = size;
    header.hasDeletions = keepDeletions;
    new HeaderSerializer().write(header, new LittleEndianDataOutputStream(out));
    fileOut.sync();
    out.close();
}
 

public TempFileIterator(
        Path tempPath,
        int tmpCount,
        Serializer<K> keySerializer
) throws IOException {
    this.tmpCount = tmpCount;
    this.keySerializer = keySerializer;
    in = new LittleEndianDataInputStream(new BufferedInputStream(Files.newInputStream(tempPath), 131072));
}
 

public Builder(
        final File file,
        final Serializer<E> serializer,
        final CompressionCodec codec
) {
    this.file = file;
    this.serializer = serializer;
    this.codec = codec;
}
 

public Reader(Path path, Comparator<K> comparator, Serializer<K> keySerializer, Serializer<V> valueSerializer, final boolean mlockFiles) throws IOException {
    this.comparator = comparator;
    indexFile = path.resolve("index.bin");
    sizeInBytes = Files.size(indexFile);
    buffer = new MMapBuffer(indexFile, FileChannel.MapMode.READ_ONLY, ByteOrder.LITTLE_ENDIAN);
    try {
        stuffToClose = SharedReference.create((Closeable)buffer);
        final MemoryDataInput in = new MemoryDataInput(buffer.memory());
        if (sizeInBytes < Header.length()) {
            throw new IOException("file is less than header length bytes");
        }
        final byte[] headerBytes = new byte[Header.length()];
        in.seek(sizeInBytes - Header.length());
        in.readFully(headerBytes);
        final LittleEndianDataInputStream headerStream = new LittleEndianDataInputStream(new ByteArrayInputStream(headerBytes));
        final Header header = new HeaderSerializer().read(headerStream);
        hasDeletions = header.hasDeletions;
        size = header.size;
        if (header.fileLength != sizeInBytes) {
            log.error(header.fileLength);
            throw new IOException("file length written to last 8 bytes of file does not match file length, file is inconsistent");
        }
        rootLevel = Level.build(buffer.memory(), keySerializer, valueSerializer, comparator, header.hasDeletions, header.indexLevels);
        rootLevelStartAddress = header.rootLevelStartAddress;
        if (mlockFiles) buffer.mlock(0, buffer.memory().length());
    } catch (Throwable t) {
        Closeables2.closeQuietly(buffer, log);
        Throwables.propagateIfInstanceOf(t, IOException.class);
        throw Throwables.propagate(t);
    }
}
 

public static <E> Writer<E> create(
        File file,
        Serializer<E> serializer,
        CompressionCodec codec,
        final long rollFrequency,
        int blockSize,
        int fileIndexBits,
        int recordIndexBits,
        int padBits,
        int maxSegment
) throws IOException {
    return new Writer(file, serializer, codec, rollFrequency, blockSize, fileIndexBits, recordIndexBits, padBits, maxSegment);
}
 

public ReplicatingStoreBuilder(File recordsPath, File indexPath, Serializer<K> keySerializer, Serializer<V> valueSerializer) {
    storeBuilder = new StoreBuilder<K, V>(indexPath, keySerializer, valueSerializer);
    this.keySerializer = keySerializer;
    this.valueSerializer = valueSerializer;
    this.recordsPath = recordsPath;
    this.indexPath = indexPath;
}
 

static <K, V> Level<K, V> build(Memory memory, Serializer<K> keySerializer, Serializer<V> valueSerializer, Comparator<K> comparator, boolean hasDeletions, int numLevels) {
    if (numLevels == 0) {
        return new Level<K, V>(memory, null, keySerializer, valueSerializer, comparator, hasDeletions);
    } else {
        return new Level<K, V>(memory, build(memory, keySerializer, valueSerializer, comparator, hasDeletions, numLevels - 1), keySerializer, new LongSerializer(), comparator, false);
    }
}
 

Level(Memory memory, @Nullable Level<K, V> nextLevel, Serializer<K> keySerializer, Serializer valueSerializer, Comparator<K> comparator, boolean hasDeletions) {
    this.memory = memory;
    this.nextLevel = nextLevel;
    this.keySerializer = keySerializer;
    this.valueSerializer = valueSerializer;
    this.comparator = comparator;
    this.hasDeletions = hasDeletions;
}
 

public RecordLogDirectory(
        File dir,
        Serializer<E> serializer,
        int maxCachedFiles,
        CompressionCodec codec,
        int blockSize,
        int fileIndexBits,
        int recordIndexBits,
        int padBits
) {
    this(dir, serializer, maxCachedFiles, codec, blockSize, fileIndexBits, recordIndexBits, padBits, false);
}
 

public static <K,V> Generation<K,V> open(BloomFilter.MemoryManager memoryManager, File file, Comparator<K> comparator, Serializer<K> keySerializer, Serializer<V> valueSerializer, StorageType storageType, CompressionCodec codec, final boolean mlockBTree)
        throws IOException {
    if (storageType == StorageType.BLOCK_COMPRESSED && codec == null) throw new IllegalArgumentException("codec must be set if block compressed");
    if (storageType == StorageType.INLINE) {
        return new InlineStableGeneration<K, V>(memoryManager, file, comparator, keySerializer, valueSerializer, mlockBTree);
    } else if (storageType == StorageType.BLOCK_COMPRESSED) {
        return new BlockCompressedStableGeneration<K, V>(memoryManager, file, comparator, keySerializer, valueSerializer, codec, mlockBTree);
    } else {
        throw new IllegalArgumentException(storageType+" is not a valid storage type");
    }
}
 

public InlineStableGeneration(BloomFilter.MemoryManager memoryManager, File file, Comparator<K> comparator,  Serializer<K> keySerializer, Serializer<V> valueSerializer, final boolean mlockBTree) throws IOException {
    this.file = file;
    reader = new ImmutableBTreeIndex.Reader(file, comparator, keySerializer, valueSerializer, mlockBTree);
    final File bloomFilterFile = new File(file, "bloomfilter.bin");
    if (bloomFilterFile.exists()) {
        bloomFilter = new BloomFilter.Reader(memoryManager, bloomFilterFile, keySerializer);
    } else {
        bloomFilter = null;
    }
}
 

public Writer(SyncableDataOutput out, Serializer<E> serializer, CompressionCodec codec, int blockSize, int recordIndexBits, int padBits) {
    if (blockSize > 1024*1024*16) throw new IllegalArgumentException("block size must be less than 2^24");
    this.out = out;
    lengthBuffer = new int[1<<recordIndexBits];
    currentBlockBytes = new UnsafeByteArrayOutputStream(blockSize);
    currentBlockOut = new DataOutputStream(currentBlockBytes);
    pad = 1<<padBits;
    shift = Math.max(recordIndexBits - padBits, 0);
    this.serializer = serializer;
    this.codec = codec;
    this.blockSize = blockSize;
}
 

private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
    final String klassName = (String) in.readObject();
    final Class klass = Class.forName(klassName);
    try {
        final Constructor<?> konstruct = klass.getConstructor();
        serializer = (Serializer<T>) konstruct.newInstance();
    } catch (final NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
        throw new IOException("can't construct wrapped serializer: " + klassName, e);
    }
}
 

public Builder(final File file, final Serializer<E> serializer) {
    this.file = file;
    this.serializer = serializer;
}
 

public Builder<E> setSerializer(final Serializer<E> serializer) {
    this.serializer = serializer;
    return this;
}
 

public GenericRecordLogAppender(File file,
                                Serializer<T> serializer,
                                CompressionCodec codec,
                                AtomicReference<Map<String, String>> metadataRef) throws IOException {
    this(file, serializer, codec, metadataRef, Long.MAX_VALUE);
}
 

public static <E> Writer<E> open(File file, Serializer<E> serializer, CompressionCodec codec, int blockSize, int recordIndexBits, int padBits) throws FileNotFoundException {
    final SyncableDataOutput out = new BufferedFileDataOutputStream(file, ByteOrder.BIG_ENDIAN, 16384);
    return new Writer<E>(out, serializer, codec, blockSize, recordIndexBits, padBits);
}
 

public ListSerializer(Supplier<List<T>> listSupplier,Serializer<T> tSerializer) {
    this.listSupplier = listSupplier;
    this.tSerializer = tSerializer;
}
 

public Reader(MemoryManager memoryManager, File file, Serializer<K> keySerializer) throws IOException {
    this.keySerializer = keySerializer;
    size = file.length();
    this.addressSpace = memoryManager.openReadOnly(file, size);
}
 

public MapSerializer(Supplier<? extends Map<K,V>> mapSupplier, Serializer<K> keySerializer, Serializer<V> valueSerializer) {
    this.mapSupplier = mapSupplier;
    this.keySerializer = keySerializer;
    this.valueSerializer = valueSerializer;
}
 

public static <K extends Comparable,V> NavigableMapSerializer<K,V> treeMapSerializer(Serializer<K> keySerializer, Serializer<V> valueSerializer) {
    return new NavigableMapSerializer<K, V>(new CollectionSuppliers.TreeMapSupplier<K, V>(), keySerializer, valueSerializer);
}
 

public NavigableMapSerializer(Supplier<? extends NavigableMap<K,V>> mapSupplier, Serializer<K> keySerializer, Serializer<V> valueSerializer) {
    mapSerializer = new MapSerializer<K, V>(mapSupplier, keySerializer, valueSerializer);
}