Java Code Examples for com.indeed.util.core.reference.SharedReference#get()

public void checkpoint(File checkpointDir) throws IOException {
    final SharedReference<GenerationState<K, V>> localState = generationState.getCopy();
    try {
        if (localState == null) {
            throw new IOException("store is closed");
        }
        checkpointDir.mkdirs();
        localState.get().volatileGeneration.checkpoint(checkpointDir);
        for (Generation<K, V> generation : localState.get().stableGenerations) {
            generation.checkpoint(checkpointDir);
        }
        PosixFileOperations.cplr(new File(localState.get().path, "state"), checkpointDir);
    } finally {
        Closeables2.closeQuietly(localState, log);
    }
}
 

public void compact() throws IOException {
    lock.lock();
    try {
        if (!closed) {
            final SharedReference<GenerationState<K, V>> localStateReference = generationState.getCopy();
            try {
                if (localStateReference == null) return;
                final GenerationState<K, V> localState = localStateReference.get();
                //this is double checked locking but in this case it doesn't really matter since it's just a heuristic
                if (localState.volatileGeneration.sizeInBytes() > maxVolatileGenerationSize) {
                    final GenerationState<K, V> nextState = startNewLog(localState);
                    startCompaction(nextState);
                }
            } finally {
                Closeables2.closeQuietly(localStateReference, log);
            }
        }
    } finally {
        lock.unlock();
    }
}
 

private static boolean shouldReloadShard(SharedReference<Shard> ref,
                                         String canonicalShardDir,
                                         long shardVersion) {
    if (ref == null)
        return true;
    final Shard oldReader = ref.get();
    return shardVersion > oldReader.getShardVersion()
            || (shardVersion == oldReader.getShardVersion() && (!canonicalShardDir.equals(oldReader.getIndexDir())));
}
 

private List<ShardInfo> buildShardList() throws IOException {
    final Map<String, Map<String, AtomicSharedReference<Shard>>> localShards = shards;
    final List<ShardInfo> ret = new ArrayList<ShardInfo>();
    for (final Map<String, AtomicSharedReference<Shard>> map : localShards.values()) {
        for (final String shardName : map.keySet()) {
            final SharedReference<Shard> ref = map.get(shardName).get();
            try {
                if (ref != null) {
                    final Shard shard = ref.get();
                    ret.add(new ShardInfo(shard.getDataset(), shardName,
                                          shard.getLoadedMetrics(), shard.getNumDocs(),
                                          shard.getShardVersion()));
                }
            } finally {
                Closeables2.closeQuietly(ref, log);
            }
        }
    }
    Collections.sort(ret, new Comparator<ShardInfo>() {
        @Override
        public int compare(ShardInfo o1, ShardInfo o2) {
            final int c = o1.dataset.compareTo(o2.dataset);
            if (c != 0)
                return c;
            return o1.shardId.compareTo(o2.shardId);
        }
    });
    return ret;
}
 

private List<DatasetInfo> buildDatasetList() throws IOException {
    final Map<String, Map<String, AtomicSharedReference<Shard>>> localShards = shards;
    final List<DatasetInfo> ret = Lists.newArrayList();
    for (final Map.Entry<String, Map<String, AtomicSharedReference<Shard>>> e : localShards.entrySet()) {
        final String dataset = e.getKey();
        final Map<String, AtomicSharedReference<Shard>> map = e.getValue();
        final List<ShardInfo> shardList = Lists.newArrayList();
        final Set<String> intFields = Sets.newHashSet();
        final Set<String> stringFields = Sets.newHashSet();
        final Set<String> metrics = Sets.newHashSet();
        for (final String shardName : map.keySet()) {
            final SharedReference<Shard> ref = map.get(shardName).get();
            try {
                if (ref != null) {
                    final Shard shard = ref.get();
                    shardList.add(new ShardInfo(shard.getDataset(), shardName,
                                                shard.getLoadedMetrics(), shard.getNumDocs(),
                                                shard.getShardVersion()));
                    intFields.addAll(shard.getIntFields());
                    stringFields.addAll(shard.getStringFields());
                    metrics.addAll(shard.getAvailableMetrics());
                }
            } finally {
                Closeables2.closeQuietly(ref, log);
            }
        }
        ret.add(new DatasetInfo(dataset, shardList, intFields, stringFields, metrics));
    }
    return ret;
}
 

private static boolean shouldReloadShard(SharedReference<Shard> ref,
                                         String canonicalShardDir,
                                         long shardVersion) {
    if (ref == null) {
        return true;
    }
    final Shard oldReader = ref.get();
    return shardVersion > oldReader.getShardVersion()
            || (shardVersion == oldReader.getShardVersion() && (!canonicalShardDir.equals(oldReader.getIndexDir())));
}
 

private List<ShardInfo> buildShardList() throws IOException {
    final Map<String, Map<String, AtomicSharedReference<Shard>>> localShards = shards;
    final List<ShardInfo> ret = new ArrayList<ShardInfo>();
    for (final Map<String, AtomicSharedReference<Shard>> map : localShards.values()) {
        for (final String shardName : map.keySet()) {
            final SharedReference<Shard> ref = map.get(shardName).getCopy();
            try {
                if (ref != null) {
                    final Shard shard = ref.get();
                    ret.add(new ShardInfo(shard.getDataset(), shardName,
                                          shard.getLoadedMetrics(), shard.getNumDocs(),
                                          shard.getShardVersion()));
                }
            } finally {
                Closeables2.closeQuietly(ref, log);
            }
        }
    }
    Collections.sort(ret, new Comparator<ShardInfo>() {
        @Override
        public int compare(ShardInfo o1, ShardInfo o2) {
            final int c = o1.dataset.compareTo(o2.dataset);
            if (c != 0) {
                return c;
            }
            return o1.shardId.compareTo(o2.shardId);
        }
    });
    return ret;
}
 

private List<DatasetInfo> buildDatasetList() throws IOException {
    final Map<String, Map<String, AtomicSharedReference<Shard>>> localShards = shards;
    final List<DatasetInfo> ret = Lists.newArrayList();
    for (final Map.Entry<String, Map<String, AtomicSharedReference<Shard>>> e : localShards.entrySet()) {
        final String dataset = e.getKey();
        final Map<String, AtomicSharedReference<Shard>> map = e.getValue();
        final List<ShardInfo> shardList = Lists.newArrayList();
        final Set<String> intFields = Sets.newHashSet();
        final Set<String> stringFields = Sets.newHashSet();
        final Set<String> metrics = Sets.newHashSet();
        for (final String shardName : map.keySet()) {
            final SharedReference<Shard> ref = map.get(shardName).getCopy();
            try {
                if (ref != null) {
                    final Shard shard = ref.get();
                    shardList.add(new ShardInfo(shard.getDataset(), shardName,
                                                shard.getLoadedMetrics(), shard.getNumDocs(),
                                                shard.getShardVersion()));
                    intFields.addAll(shard.getIntFields());
                    stringFields.addAll(shard.getStringFields());
                    metrics.addAll(shard.getAvailableMetrics());
                }
            } finally {
                Closeables2.closeQuietly(ref, log);
            }
        }
        ret.add(new DatasetInfo(dataset, shardList, intFields, stringFields, metrics));
    }
    return ret;
}
 

private void startCompaction(final GenerationState<K, V> localState) throws IOException {
    //find generations eligible for compaction and start compaction in background
    final List<SharedReference<? extends Generation<K,V>>> toCompact = Lists.newArrayList();
    long sum = 0;
    boolean hasDeletions = false;
    for (SharedReference<? extends Generation<K, V>> reference : localState.stableGenerationReferences) {
        final Generation<K, V> generation = reference.get();
        final String name = generation.getPath().getName();
        if (!currentlyCompacting.contains(name)) {
            if ((generation instanceof VolatileGeneration || (sum*2 > generation.sizeInBytes()))) {
                sum+=generation.sizeInBytes();
                toCompact.add(reference.copy());
                currentlyCompacting.add(generation.getPath().getName());
            } else {
                hasDeletions = true;
                break;
            }
        } else {
            hasDeletions = true;
            break;
        }
    }
    if (toCompact.size() > 0) {
        runningCompactions++;
        threadPool.execute(new Compaction(toCompact, hasDeletions));
    }
}
 

private void finishCompaction(final Set<String> compactedGenerations, final List<Generation<K, V>> toCompactGenerations, final Generation<K, V> stableGeneration) throws IOException {
    final List<SharedReference<? extends Generation<K,V>>> nextStableGenerations = Lists.newArrayList();
    final SharedReference<GenerationState<K, V>> stateReference = Preconditions.checkNotNull(generationState.getCopy());
    final GenerationState<K,V> state = stateReference.get();
    try {
        boolean compactionAdded = false;
        for (SharedReference<? extends Generation<K, V>> reference : state.stableGenerationReferences) {
            final String name = reference.get().getPath().getName();
            if (!compactedGenerations.contains(name)) {
                nextStableGenerations.add(reference.copy());
            } else {
                if (!compactionAdded) {
                    nextStableGenerations.add(SharedReference.create(stableGeneration));
                    compactionAdded = true;
                }
                currentlyCompacting.remove(name);
            }
        }
        final File checkpointDir = getNextCheckpointDir();
        checkpointDir.mkdirs();
        final GenerationState<K,V> nextState = new GenerationState<K, V>(nextStableGenerations, state.volatileGenerationReference.copy(), checkpointDir);
        checkpointGenerationState(nextState, checkpointDir);
        PosixFileOperations.atomicLink(checkpointDir, new File(root, "latest"));
        final SharedReference<GenerationState<K, V>> oldState = Preconditions.checkNotNull(generationState.getAndSet(nextState));
        oldState.get().delete();
        Closeables2.closeQuietly(oldState, log);
        for (Generation<K, V> generation : toCompactGenerations) {
            final long sizeInBytes = generation.sizeInBytes();
            generation.delete();
            totalGenerationSpace.addAndGet(-sizeInBytes);
        }
    } finally {
        Closeables2.closeQuietly(stateReference, log);
    }
}
 

private void finishClose() throws IOException {
    try {
        final SharedReference<GenerationState<K, V>> state = generationState.getAndUnset();
        try {
            if (state != null) {
                final VolatileGeneration<K, V> volatileGeneration = state.get().volatileGeneration;
                if (volatileGeneration != null) volatileGeneration.closeWriter();
            }
        } finally {
            Closeables2.closeQuietly(state, log);
        }
    } finally {
        threadPool.shutdown();
    }
}
 

public GenerationState(
        final List<SharedReference<? extends Generation<K, V>>> stableGenerationReferences,
        final SharedReference<VolatileGeneration<K, V>> volatileGenerationReference,
        File path
) {
    this.path = path;
    this.stableGenerationReferences = ImmutableList.copyOf(stableGenerationReferences);
    this.volatileGenerationReference = volatileGenerationReference;
    this.volatileGeneration = volatileGenerationReference.get();
    final ImmutableList.Builder<Generation<K,V>> builder = ImmutableList.builder();
    for (SharedReference<? extends Generation<K, V>> generation : stableGenerationReferences) {
        builder.add(generation.get());
    }
    stableGenerations = builder.build();
}
 

public CachedFlamdexReaderReference(final SharedReference<? extends CachedFlamdexReader> reference) {
    constructorStackTrace = new Exception();
    this.reference = reference;
    this.reader = reference.get();
}
 

public RawCachedFlamdexReaderReference(final SharedReference<? extends RawCachedFlamdexReader> reference) {
    super(reference);
    this.reader = reference.get();
}
 

private CachedIntValueLookup(final SharedReference<IntValueLookup> reference) {
    this.reference = reference;
    metric = reference.get();
}