Java Code Examples for com.sleepycat.je.DatabaseConfig#setReadOnly()

public static Map<String,Object> getDatabaseStatistics(Environment environment, String database, boolean reset)
{
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setReadOnly(true);
    DbInternal.setUseExistingConfig(dbConfig, true);
    try (Database db = environment.openDatabase(null, database, dbConfig))
    {
        StatsConfig config = new StatsConfig();
        config.setClear(reset);
        config.setFast(false);

        BtreeStats stats = (BtreeStats) db.getStats(config);

        Map<String, Object> results = new TreeMap<>();
        results.put(BTREE_BIN_COUNT.getName(), stats.getBottomInternalNodeCount());
        results.put(BTREE_DELETED_LN_COUNT.getName(), stats.getDeletedLeafNodeCount());
        results.put(BTREE_IN_COUNT.getName(), stats.getInternalNodeCount());
        results.put(BTREE_LN_COUNT.getName(), stats.getLeafNodeCount());
        results.put(BTREE_MAINTREE_MAXDEPTH.getName(), stats.getMainTreeMaxDepth());
        results.put(BTREE_INS_BYLEVEL.getName(), Arrays.asList(stats.getINsByLevel()));
        results.put(BTREE_BINS_BYLEVEL.getName(), Arrays.asList(stats.getBINsByLevel()));
        results.put(BTREE_BIN_ENTRIES_HISTOGRAM.getName(), Arrays.asList(stats.getBINEntriesHistogram()));
        results.put(BTREE_RELATCHES_REQUIRED.getName(), stats.getRelatches());
        results.put(BTREE_ROOT_SPLITS.getName(), stats.getRootSplits());

        return results;
    }

}
 

/**
 * This method creates and/or opens the BDB databases with the appropriate parameters.
 * 
 * @throws Exception
 */
private void createOrOpenBDBDbs() throws Exception {
	// configuration for the mapping dbs
	DatabaseConfig dbConfig = new DatabaseConfig();
	dbConfig.setAllowCreate(true); // db will be created if it does not exist
	dbConfig.setReadOnly(readOnly);
	dbConfig.setTransactional(transactional);
	// create/open mapping dbs using config
	iidToIdDB = dbEnv.openDatabase(null, "idToName", dbConfig);

	// if countSizeOnLoad is true, the id-name mappings are counted and the loadCounter is initialized
	if (countSizeOnLoad) {
		System.out.println(new Date() + " counting index size started ");
		int idToNameMappings = (int) iidToIdDB.count();
		loadCounter = Math.min(idToNameMappings, maxNumVectors);
		System.out.println(new Date() + " counting index size ended ");
		System.out.println("Index size: " + loadCounter);
	}

	idToIidDB = dbEnv.openDatabase(null, "nameToId", dbConfig);

	if (useGeolocation) {// create/open geolocation db using config
		iidToGeolocationDB = dbEnv.openDatabase(null, "idToGeolocation", dbConfig);
	}

	if (useMetaData) {
		StoreConfig storeConfig = new StoreConfig(); // configuration of the entity store
		storeConfig.setAllowCreate(true); // store will be created if it does not exist
		storeConfig.setReadOnly(readOnly);
		storeConfig.setTransactional(transactional);
		iidToMetadataDB = new EntityStore(dbEnv, "idToMetadata", storeConfig);
		// int nameToMetadataMappings = (int) nameToMetadataBDB.getPrimaryIndex(String.class,
		// MediaFeedData.class).count(); // counting the size of an EntityStore
	}
}
 

/**
 * Advanced constructor.
 * 
 * @param vectorLength
 *            The dimensionality of the VLAD vectors being indexed
 * @param maxNumVectors
 *            The maximum allowable size (number of vectors) of the index
 * @param readOnly
 *            If true the persistent store will opened only for read access (allows multiple opens)
 * @param BDBEnvHome
 *            The BDB environment home directory
 * @param numSubVectors
 *            The number of subvectors
 * @param numProductCentroids
 *            The number of centroids used to quantize each sub-vector
 * @param transformation
 *            The type of transformation to perform on each vector
 * @param countSizeOnLoad
 *            Whether the load counter will be initialized by the size of the persistent store
 * @param loadCounter
 *            The initial value of the load counter
 * @param loadIndexInMemory
 *            Whether to load the index in memory, we can avoid loading the index in memory when we only
 *            want to perform indexing
 * @throws Exception
 */
public PQ(int vectorLength, int maxNumVectors, boolean readOnly, String BDBEnvHome, int numSubVectors,
		int numProductCentroids, TransformationType transformation, boolean countSizeOnLoad,
		int loadCounter, boolean loadIndexInMemory, long cacheSize) throws Exception {
	super(vectorLength, maxNumVectors, readOnly, countSizeOnLoad, loadCounter, loadIndexInMemory,
			cacheSize);
	this.numSubVectors = numSubVectors;
	if (vectorLength % numSubVectors > 0) {
		throw new Exception("The given number of subvectors is not valid!");
	}
	this.subVectorLength = vectorLength / numSubVectors;
	this.numProductCentroids = numProductCentroids;
	this.transformation = transformation;

	if (transformation == TransformationType.RandomRotation) {
		this.rr = new RandomRotation(seed, vectorLength);
	} else if (transformation == TransformationType.RandomPermutation) {
		this.rp = new RandomPermutation(seed, vectorLength);
	}

	createOrOpenBDBEnvAndDbs(BDBEnvHome);

	// configuration of the persistent index
	DatabaseConfig dbConf = new DatabaseConfig();
	dbConf.setReadOnly(readOnly);
	dbConf.setTransactional(transactional);
	dbConf.setAllowCreate(true); // db will be created if it does not exist
	iidToPqDB = dbEnv.openDatabase(null, "adc", dbConf); // create/open the db using config

	if (loadIndexInMemory) {
		// initialize the in-memory data structures and load any existing persistent index in memory
		loadIndexInMemory();
	}
}
 

/**
 * Advanced constructor.
 * 
 * @param vectorLength
 *            The dimensionality of the VLAD vectors being indexed
 * @param maxNumVectors
 *            The maximum allowable size (number of vectors) of the index
 * @param readOnly
 *            If true the persistent store will opened only for read access (allows multiple opens)
 * @param BDBEnvHome
 *            The BDB environment home directory
 * @param numSubVectors
 *            The number of subvectors
 * @param numProductCentroids
 *            The number of centroids used to quantize each sub-vector
 * @param transformation
 *            The type of transformation to perform on each vector
 * @param numCoarseCentroids
 *            The number of centroids of the coarse quantizer
 * @param countSizeOnLoad
 *            Whether the load counter will be initialized by the size of the persistent store
 * @param loadCounter
 *            The initial value of the load counter
 * @param loadIndexInMemory
 *            Whether to load the index in memory, we can avoid loading the index in memory when we only
 *            want to perform indexing
 * @param cacheSize
 *            the size of the cache in Megabytes
 * @throws Exception
 */
public IVFPQ(int vectorLength, int maxNumVectors, boolean readOnly, String BDBEnvHome, int numSubVectors,
		int numProductCentroids, TransformationType transformation, int numCoarseCentroids,
		boolean countSizeOnLoad, int loadCounter, boolean loadIndexInMemory, long cacheSize)
				throws Exception {
	super(vectorLength, maxNumVectors, readOnly, countSizeOnLoad, loadCounter, loadIndexInMemory,
			cacheSize);
	this.numSubVectors = numSubVectors;
	if (vectorLength % numSubVectors > 0) {
		throw new Exception("The given number of subvectors is not valid!");
	}
	this.subVectorLength = vectorLength / numSubVectors;
	this.numProductCentroids = numProductCentroids;
	this.transformation = transformation;
	this.numCoarseCentroids = numCoarseCentroids;
	w = (int) (numCoarseCentroids * 0.1); // by default set w to 10% of the lists

	if (transformation == TransformationType.RandomRotation) {
		this.rr = new RandomRotation(seed, vectorLength);
	} else if (transformation == TransformationType.RandomPermutation) {
		this.rp = new RandomPermutation(seed, vectorLength);
	}

	createOrOpenBDBEnvAndDbs(BDBEnvHome);

	// configuration of the persistent index
	DatabaseConfig dbConf = new DatabaseConfig();
	dbConf.setReadOnly(readOnly);
	dbConf.setTransactional(transactional);
	dbConf.setAllowCreate(true); // db will be created if it does not exist
	iidToIvfpqDB = dbEnv.openDatabase(null, "ivfadc", dbConf); // create/open the db using config

	if (loadIndexInMemory) {// load the existing persistent index in memory
		// create the memory objects with the appropriate initial size
		invertedLists = new TIntArrayList[numCoarseCentroids];

		if (numProductCentroids <= 256) {
			pqByteCodes = new TByteArrayList[numCoarseCentroids];
		} else {
			pqShortCodes = new TShortArrayList[numCoarseCentroids];
		}

		int initialListCapacity = (int) ((double) maxNumVectors / numCoarseCentroids);
		System.out.println("Calculated list size " + initialListCapacity);

		for (int i = 0; i < numCoarseCentroids; i++) {
			if (numProductCentroids <= 256) {
				// fixed initial size allows space efficiency measurements
				// pqByteCodes[i] = new TByteArrayList(initialListCapacity * numSubVectors);
				pqByteCodes[i] = new TByteArrayList();

			} else {
				// fixed initial size allows space efficiency measurements
				// pqShortCodes[i] = new TShortArrayList(initialListCapacity * numSubVectors);
				pqShortCodes[i] = new TShortArrayList();

			}
			// fixed initial size for each list, allows space efficiency measurements
			// invertedLists[i] = new TIntArrayList(initialListCapacity);
			invertedLists[i] = new TIntArrayList();
		}
		// load any existing persistent index in memory
		loadIndexInMemory();
	}
}
 

/**
 * Advanced constructor.
 * 
 * @param vectorLength
 *            The dimensionality of the VLAD vectors being indexed
 * @param maxNumVectors
 *            The maximum allowable size (number of vectors) of the index
 * @param readOnly
 *            If true the persistent store will opened only for read access (allows multiple opens)
 * @param BDBEnvHome
 *            The BDB environment home directory
 * @param loadIndexInMemory
 *            Whether to load the index in memory, we can avoid loading the index in memory when we only
 *            want to perform indexing
 * @param countSizeOnLoad
 *            Whether the load counter will be initialized by the size of the persistent store
 * @param loadCounter
 *            The initial value of the load counter
 * @throws Exception
 */
public Linear(int vectorLength, int maxNumVectors, boolean readOnly, String BDBEnvHome,
		boolean loadIndexInMemory, boolean countSizeOnLoad, int loadCounter) throws Exception {
	super(vectorLength, maxNumVectors, readOnly, countSizeOnLoad, loadCounter, loadIndexInMemory);
	createOrOpenBDBEnvAndDbs(BDBEnvHome);
	// configuration of the persistent index
	DatabaseConfig dbConf = new DatabaseConfig();
	dbConf.setReadOnly(readOnly);
	dbConf.setTransactional(transactional);
	dbConf.setAllowCreate(true); // db will be created if it does not exist
	iidToVectorDB = dbEnv.openDatabase(null, "vlad", dbConf); // create/open the db using config

	if (loadIndexInMemory) {// load the existing persistent index in memory
		// create the memory objects with the appropriate initial size
		vectorsList = new TDoubleArrayList(maxNumVectors * vectorLength);
		loadIndexInMemory();
	}
}