com.sleepycat.je.Environment Java Examples

public static Map<String,Object> getTransactionStatistics(Environment environment, boolean reset)
{
    StatsConfig config = new StatsConfig();
    config.setClear(reset);
    config.setFast(false);
    final TransactionStats stats = environment.getTransactionStats(config);
    Map<String,Object> results = new LinkedHashMap<>();

    results.put(TXN_ACTIVE.getName(), stats.getNActive());
    results.put(TXN_BEGINS.getName(), stats.getNBegins());
    results.put(TXN_COMMITS.getName(), stats.getNCommits());
    results.put(TXN_ABORTS.getName(), stats.getNAborts());
    results.put(TXN_XAPREPARES.getName(), stats.getNXAPrepares());
    results.put(TXN_XACOMMITS.getName(), stats.getNXACommits());
    results.put(TXN_XAABORTS.getName(), stats.getNXAAborts());

    return results;
}
 

private List<AMQShortString> findTopicExchanges(final Environment environment)
{
    final List<AMQShortString> topicExchanges = new ArrayList<AMQShortString>();
    final ExchangeRecordBinding binding = new ExchangeRecordBinding();
    CursorOperation databaseOperation = new CursorOperation()
    {

        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction,
                DatabaseEntry key, DatabaseEntry value)
        {
            ExchangeRecord record = binding.entryToObject(value);
            if (AMQShortString.valueOf(ExchangeDefaults.TOPIC_EXCHANGE_CLASS).equals(record.getType()))
            {
                topicExchanges.add(record.getName());
            }
        }
    };
    new DatabaseTemplate(environment, EXCHANGE_DB_NAME, null).run(databaseOperation);
    return topicExchanges;
}
 

@Override
public void performUpgrade(final Environment environment, final UpgradeInteractionHandler handler, ConfiguredObject<?> parent)
{
    Transaction transaction = null;
    reportStarting(environment, 4);

    transaction = environment.beginTransaction(null, null);

    // find all queues which are bound to a topic exchange and which have a colon in their name
    final List<AMQShortString> potentialDurableSubs = findPotentialDurableSubscriptions(environment, transaction);

    Set<String> existingQueues = upgradeQueues(environment, handler, potentialDurableSubs, transaction);
    upgradeQueueBindings(environment, handler, potentialDurableSubs, transaction);
    Set<Long> messagesToDiscard = upgradeDelivery(environment, existingQueues, handler, transaction);
    upgradeContent(environment, handler, messagesToDiscard, transaction);
    upgradeMetaData(environment, handler, messagesToDiscard, transaction);
    renameRemainingDatabases(environment, handler, transaction);
    transaction.commit();

    reportFinished(environment, 5);
}
 

public void backupDatabase(Environment environment, Path dbPath, Path backupPath) throws IOException {
  final Path lastBackuppedFilePath = backupPath.resolve("lastFile");
  final DbBackup backupHelper = lastBackuppedFilePath.toFile().exists() ?
      new DbBackup(environment, lastFile(lastBackuppedFilePath)) :
      new DbBackup(environment);
  environment.sync();

  try {
    backupHelper.startBackup();
    final String[] logFilesInBackupSet = backupHelper.getLogFilesInBackupSet();
    for (String logFile : logFilesInBackupSet) {
      Files.copy(dbPath.resolve(logFile), backupPath.resolve(logFile), REPLACE_EXISTING);
    }
    final long lastFileInBackupSet = backupHelper.getLastFileInBackupSet();
    Files.write(lastBackuppedFilePath, Lists.newArrayList("" + lastFileInBackupSet), UTF_8  );
  } finally {
    backupHelper.endBackup();
  }

}
 

@Test
public void testEmptyDatabaseUpgradeDoesNothing() throws Exception
{
    File nonExistentStoreLocation = new File(TMP_FOLDER, getTestName());
    deleteDirectoryIfExists(nonExistentStoreLocation);

    nonExistentStoreLocation.mkdir();
    Environment emptyEnvironment = createEnvironment(nonExistentStoreLocation);
    try
    {
        _upgrader = new Upgrader(emptyEnvironment, getVirtualHost());
        _upgrader.upgradeIfNecessary();

        List<String> databaseNames = emptyEnvironment.getDatabaseNames();
        List<String> expectedDatabases = new ArrayList<String>();
        expectedDatabases.add(Upgrader.VERSION_DB_NAME);
        assertEquals("Expectedonly VERSION table in initially empty store after upgrade: ", expectedDatabases, databaseNames);
        assertEquals("Unexpected store version", BDBConfigurationStore.VERSION, getStoreVersion(emptyEnvironment));

    }
    finally
    {
        emptyEnvironment.close();
        nonExistentStoreLocation.delete();
    }
}
 

private int getStoreVersion(Environment environment)
{
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setTransactional(true);
    dbConfig.setAllowCreate(true);
    int storeVersion = -1;
    try(Database versionDb = environment.openDatabase(null, Upgrader.VERSION_DB_NAME, dbConfig);
        Cursor cursor = versionDb.openCursor(null, null))
    {
        DatabaseEntry key = new DatabaseEntry();
        DatabaseEntry value = new DatabaseEntry();
        while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS)
        {
            int version = IntegerBinding.entryToInt(key);
            if (storeVersion < version)
            {
                storeVersion = version;
            }
        }
    }
    return storeVersion;
}
 

private void renameDatabases(Environment environment, Transaction transaction)
{
    List<String> databases = environment.getDatabaseNames();
    String[] oldDatabases = { OLD_META_DATA_DB_NAME, OLD_BRIDGES_DB_NAME, OLD_LINKS_DB_NAME };
    String[] newDatabases = { NEW_METADATA_DB_NAME, NEW_BRIDGES_DB_NAME, NEW_LINKS_DB_NAME };

    for (int i = 0; i < oldDatabases.length; i++)
    {
        String oldName = oldDatabases[i];
        String newName = newDatabases[i];
        if (databases.contains(oldName))
        {
            LOGGER.info("Renaming " + oldName + " into " + newName);
            environment.renameDatabase(transaction, oldName, newName);
        }
    }
}
 

private void populateOldXidEntries(Environment environment)
{

    final DatabaseEntry value = new DatabaseEntry();
    OldRecordImpl[] enqueues = { new OldRecordImpl("TEST1", 1) };
    OldRecordImpl[] dequeues = { new OldRecordImpl("TEST2", 2) };
    OldPreparedTransaction oldPreparedTransaction = new OldPreparedTransaction(enqueues, dequeues);
    OldPreparedTransactionBinding oldPreparedTransactionBinding = new OldPreparedTransactionBinding();
    oldPreparedTransactionBinding.objectToEntry(oldPreparedTransaction, value);

    final DatabaseEntry key = getXidKey();
    new DatabaseTemplate(environment, OLD_XID_DB_NAME, null).run(new DatabaseRunnable()
    {

        @Override
        public void run(Database xidDatabase, Database nullDatabase, Transaction transaction)
        {
            xidDatabase.put(null, key, value);
        }
    });
}
 

public TestKDTreeSplit(final File tmpDir, final List<Pair<String, String>> filesAndFormats,
		final List<Integer> experimentSize) {

	this.filesAndFormats = filesAndFormats;
	this.experimentSize = experimentSize;

	this.elements = new HashMap<>();
	this.elementCounter = new HashMap<>();
	this.boxDimension = new HashMap<>();

	// Setup database dir
	tmpDir.mkdirs();
	FileUtil.deleteDirOnExit(tmpDir.toPath());

	final EnvironmentConfig envConfig = new EnvironmentConfig();
	envConfig.setTransactional(false);
	envConfig.setAllowCreate(true);
    envConfig.setSharedCache(true);
	dbEnv = new Environment(tmpDir, envConfig);

	dbConfig = new DatabaseConfig();
	dbConfig.setTransactional(false);
	dbConfig.setAllowCreate(true);
}
 

private void closeEnvironment()
{
    Environment environment = _environment.getAndSet(null);
    if (environment != null)
    {
        // Clean the log before closing. This makes sure it doesn't contain
        // redundant data. Closing without doing this means the cleaner may
        // not get a chance to finish.
        try
        {
            BDBUtils.runCleaner(environment);
        }
        finally
        {
            environment.close();
        }
    }
}
 

private void assertXidEntries(Environment environment)
{
    final DatabaseEntry value = new DatabaseEntry();
    final DatabaseEntry key = getXidKey();
    new DatabaseTemplate(environment, NEW_XID_DB_NAME, null).run(new DatabaseRunnable()
    {

        @Override
        public void run(Database xidDatabase, Database nullDatabase, Transaction transaction)
        {
            xidDatabase.get(null, key, value, LockMode.DEFAULT);
        }
    });
    NewPreparedTransactionBinding newBinding = new NewPreparedTransactionBinding();
    NewPreparedTransaction newTransaction = newBinding.entryToObject(value);
    NewRecordImpl[] newEnqueues = newTransaction.getEnqueues();
    NewRecordImpl[] newDequeues = newTransaction.getDequeues();
    assertEquals("Unxpected new enqueus number", 1, newEnqueues.length);
    NewRecordImpl enqueue = newEnqueues[0];
    assertEquals("Unxpected queue id", UUIDGenerator.generateQueueUUID("TEST1", getVirtualHost().getName()), enqueue.getId());
    assertEquals("Unxpected message id", 1, enqueue.getMessageNumber());
    assertEquals("Unxpected new dequeues number", 1, newDequeues.length);
    NewRecordImpl dequeue = newDequeues[0];
    assertEquals("Unxpected queue id", UUIDGenerator.generateQueueUUID("TEST2", getVirtualHost().getName()), dequeue.getId());
    assertEquals("Unxpected message id", 2, dequeue.getMessageNumber());
}
 

/**
 * Open the Berkeley DB
 * @throws IOException
 */
protected void openDatabase() throws IOException {
	final EnvironmentConfig envConfig = new EnvironmentConfig();
	envConfig.setTransactional(false);
	envConfig.setAllowCreate(true);
    envConfig.setSharedCache(true);

    tmpDatabaseDir = Files.createTempDirectory(null);
	dbEnv = new Environment(tmpDatabaseDir.toFile(), envConfig);

	logger.info("Database dir is {}", tmpDatabaseDir);

	// Delete database on exit
	FileUtil.deleteDirOnExit(tmpDatabaseDir);

	final DatabaseConfig dbConfig = new DatabaseConfig();
	dbConfig.setTransactional(false);
	dbConfig.setAllowCreate(true);
	dbConfig.setDeferredWrite(true);

	database = dbEnv.openDatabase(null, "lines", dbConfig);
}
 

public Database getDatabase(String name, boolean isTmp, boolean isSortedDuplicates) {
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setAllowCreate(true);
    Environment _env = env;
    if (isTmp) {
        dbConfig.setTemporary(true);
        dbConfig.setSortedDuplicates(isSortedDuplicates);
        _env = getTmpEnv();
    } else {
        if (!config.isTransactional()) {
            dbConfig.setDeferredWrite(config.isCommitSync());
        } else {
            dbConfig.setTransactional(true);
        }
    }

    Database database = buildPrimaryIndex(dbConfig, _env, name);
    return database;
}
 

/**
 * 构造函数
 */
public WebGraph(String dbDir) throws DatabaseException {
  File envDir = new File(dbDir);
  EnvironmentConfig envConfig = new EnvironmentConfig();
  envConfig.setTransactional(false);
  envConfig.setAllowCreate(true);
  Environment env = new Environment(envDir, envConfig);

  StoreConfig storeConfig = new StoreConfig();
  storeConfig.setAllowCreate(true);
  storeConfig.setTransactional(false);

  store = new EntityStore(env, "classDb", storeConfig);
  outLinkIndex = store.getPrimaryIndex(String.class, Link.class);
  inLinkIndex = store.getSecondaryIndex(outLinkIndex, String.class, "toURL");
}
 

private int getVersion(final Environment env, final DatabaseConfig dbConfig)
{
    try (Database versionDb = env.openDatabase(null, VERSION_DB_NAME, dbConfig);
         Cursor cursor = versionDb.openCursor(null, null))
    {
        DatabaseEntry key = new DatabaseEntry();
        DatabaseEntry value = new DatabaseEntry();

        int version = 0;

        while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS)
        {
            int ver = IntegerBinding.entryToInt(key);
            if (ver > version)
            {
                version = ver;
            }
        }

        return version;
    }
}
 

/**
 * 扫描指定环境中的实体名称与仓储名称的映射
 * 
 * @param environment
 * @return 实体名称与仓储名称的映射
 */
private Map<String, String> scanEntityStoreNameMap(Environment environment) {
    final Map<String, String> entityStoreNameMap = new HashMap<String, String>();
    for (String databaseName : environment.getDatabaseNames()) {
        // 检测数据库名称是否符合EntityStore的名称规范(persist#STORE_NAME#ENTITY_CLASS)
        if (databaseName.startsWith(PERSIST_PREFIX) && !databaseName.endsWith(FORMAT_SUFFIX) && !databaseName.endsWith(SEQUENCE_SUFFIX)) {
            final String[] databasePartNames = databaseName.split("#");
            // 防止重复构建RawStore
            if (databasePartNames.length == 3) {
                if (entityStoreNameMap.containsKey(databasePartNames[2])) {
                    throw new RuntimeException("实体名称冲突");
                } else {
                    entityStoreNameMap.put(databasePartNames[2], databasePartNames[1]);
                }
            }
        }
    }
    return entityStoreNameMap;
}
 

@Override
public <KeyT, ValueT> BdbWrapper<KeyT, ValueT> getDatabase(String userId, String dataSetId, String databaseName,
                                                           boolean allowDuplicates, EntryBinding<KeyT> keyBinder,
                                                           EntryBinding<ValueT> valueBinder,
                                                           IsCleanHandler<KeyT, ValueT> isCleanHandler)
  throws BdbDbCreationException {
  try {
    DatabaseConfig config = new DatabaseConfig();
    config.setAllowCreate(true);
    config.setDeferredWrite(true);
    config.setSortedDuplicates(allowDuplicates);

    String environmentKey = environmentKey(userId, dataSetId);
    File envHome = new File(dbHome, environmentKey);
    envHome.mkdirs();
    Environment dataSetEnvironment = new Environment(envHome, configuration);
    Database database = dataSetEnvironment.openDatabase(null, databaseName, config);
    databases.put(environmentKey + "_" + databaseName, database);
    environmentMap.put(environmentKey, dataSetEnvironment);
    return new BdbWrapper<>(dataSetEnvironment, database, config, keyBinder, valueBinder, isCleanHandler);
  } catch (DatabaseException e) {
    throw new BdbDbCreationException(e);
  }
}
 

ClassCatalog(String directory, EnvironmentConfig environmentConfig, DatabaseConfig dbConfig) {
	// This should be main database directory
	dirClassLoader = new DirClassLoader(directory);
	
    // Open the environment in subdirectory of the above
	String classesDir = directory + java.io.File.separator + "classes";
	RelDatabase.mkdir(classesDir);
    environment = new Environment(new File(classesDir), environmentConfig);
    
    // Open the class catalog db. This is used to optimize class serialization.
    classCatalogDb = environment.openDatabase(null, "_ClassCatalog", dbConfig);

    // Create our class catalog
    classCatalog = new StoredClassCatalog(classCatalogDb);
    
    // Need a serial binding for metadata
    relvarMetadataBinding = new SerialBinding<RelvarMetadata>(classCatalog, RelvarMetadata.class);
    
    // Need serial binding for data
    tupleBinding = new SerialBinding<ValueTuple>(classCatalog, ValueTuple.class) {
    	public ClassLoader getClassLoader() {
    		return dirClassLoader;
    	}
    };   	
}
 

@Override
public void init() {

    EnvironmentConfig envConfig = new EnvironmentConfig();
    commonConfig(envConfig);
    envConfig.setCachePercent(config.getCachePercent());
    envConfig.setAllowCreate(true);
    if (config.isTransactional()) {
        envConfig.setCachePercent(config.getCachePercent());
        envConfig.setTransactional(config.isTransactional());
        envConfig.setTxnTimeout(config.getTxnTimeout(), TimeUnit.SECONDS);
        this.durability = config.isCommitSync() ? Durability.COMMIT_SYNC : Durability.COMMIT_NO_SYNC;
        envConfig.setDurability(this.durability);
    }
    File repo_dir = new File(config.getRepoDir());
    if (!repo_dir.exists()) {
        repo_dir.mkdirs();
    }
    this.env = new Environment(repo_dir, envConfig);
    cef = new CommandHandlerFactoryBDBImpl();
    cursorFactoryBDBImp = new CursorFactoryBDBImp();

}
 

@Test
public void testPerformXidUpgrade() throws Exception
{
    File storeLocation = new File(TMP_FOLDER, getTestName());
    storeLocation.mkdirs();
    Environment environment = createEnvironment(storeLocation);
    try
    {
        populateOldXidEntries(environment);
        UpgradeFrom5To6 upgrade = new UpgradeFrom5To6();
        upgrade.performUpgrade(environment, UpgradeInteractionHandler.DEFAULT_HANDLER, getVirtualHost());
        assertXidEntries(environment);
    }
    finally
    {
        try
        {
            environment.close();
        }
        finally
        {
            deleteDirectoryIfExists(storeLocation);
        }

    }
}
 

public AbstractFrontier(String homeDirectory) {
  EnvironmentConfig envConfig = new EnvironmentConfig();
  envConfig.setTransactional(true);
  envConfig.setAllowCreate(true);
  env = new Environment(new File(homeDirectory), envConfig);
  // 设置databaseconfig
  DatabaseConfig dbConfig = new DatabaseConfig();
  dbConfig.setTransactional(true);
  dbConfig.setAllowCreate(true);
  // 打开
  catalogdatabase = env.openDatabase(null, CLASS_CATALOG, dbConfig);
  javaCatalog = new StoredClassCatalog(catalogdatabase);
  // 设置databaseconfig
  DatabaseConfig dbConfig0 = new DatabaseConfig();
  dbConfig0.setTransactional(true);
  dbConfig0.setAllowCreate(true);
  database = env.openDatabase(null, "URL", dbConfig0);
}
 

/**
 * Constructor
 * @param name The unique name of this manager.
 * @param shortName Original name for the Manager.
 * @param agents An array of Agents.
 * @param batchSize The number of events to include in a batch.
 * @param retries The number of times to retry connecting before giving up.
 * @param connectionTimeout The amount of time to wait for a connection to be established.
 * @param requestTimeout The amount of time to wair for a response to a request.
 * @param delay The amount of time to wait between retries.
 * @param database The database to write to.
 * @param environment The database environment.
 * @param secretKey The SecretKey to use for encryption.
 * @param lockTimeoutRetryCount The number of times to retry a lock timeout.
 */
protected FlumePersistentManager(final String name, final String shortName, final Agent[] agents,
                                 final int batchSize, final int retries, final int connectionTimeout,
                                 final int requestTimeout, final int delay, final Database database,
                                 final Environment environment, final SecretKey secretKey,
                                 final int lockTimeoutRetryCount) {
    super(name, shortName, agents, batchSize, delay, retries, connectionTimeout, requestTimeout);
    this.database = database;
    this.environment = environment;
    dbCount.set(database.count());
    this.worker = new WriterThread(database, environment, this, gate, batchSize, secretKey, dbCount,
        lockTimeoutRetryCount);
    this.worker.start();
    this.secretKey = secretKey;
    this.threadPool = Executors.newCachedThreadPool(Log4jThreadFactory.createDaemonThreadFactory("Flume"));
    this.lockTimeoutRetryCount = lockTimeoutRetryCount;
}
 

@Before
public void setUp() throws Exception
{
    _environment = mock(Environment.class);
    _sourceDatabase = mock(Database.class);
    when(_environment.openDatabase(any(Transaction.class), same(SOURCE_DATABASE), isA(DatabaseConfig.class)))
            .thenReturn(_sourceDatabase);
    when(_environment.openDatabase(isNull(), same(SOURCE_DATABASE), isA(DatabaseConfig.class)))
            .thenReturn(_sourceDatabase);
}
 

public BDBWriter(final byte[] keyData, final byte[] eventData, final Environment environment,
                 final Database database, final Gate gate, final AtomicLong dbCount, final long batchSize,
                 final int lockTimeoutRetryCount) {
    this.keyData = keyData;
    this.eventData = eventData;
    this.environment = environment;
    this.database = database;
    this.gate = gate;
    this.dbCount = dbCount;
    this.batchSize = batchSize;
    this.lockTimeoutRetryCount = lockTimeoutRetryCount;
}
 

protected Environment createEnvironment(File storeLocation)
{
    EnvironmentConfig envConfig = new EnvironmentConfig();
    envConfig.setAllowCreate(true);
    envConfig.setTransactional(true);
    envConfig.setConfigParam("je.lock.nLockTables", "7");
    envConfig.setReadOnly(false);
    envConfig.setSharedCache(false);
    envConfig.setCacheSize(0);
    return new Environment(storeLocation, envConfig);
}
 

public WriterThread(final Database database, final Environment environment,
                    final FlumePersistentManager manager, final Gate gate, final int batchsize,
                    final SecretKey secretKey, final AtomicLong dbCount, final int lockTimeoutRetryCount) {
    super("FlumePersistentManager-Writer");
    this.database = database;
    this.environment = environment;
    this.manager = manager;
    this.gate = gate;
    this.batchSize = batchsize;
    this.secretKey = secretKey;
    this.setDaemon(true);
    this.dbCounter = dbCount;
    this.lockTimeoutRetryCount = lockTimeoutRetryCount;
}
 

/**
 * This method is invoked by the kernel to initialize the storage.
 *
 * @param arguments The arguments with which this storage is to be
 *                  initialized.
 * @return True if the storage was initialized successfully.
 */
public boolean initialize(String filePath) {
	//clock = System.currentTimeMillis();
	countUpdates = 0;
	annotationsTime = 0;
	scaffoldTime = 0;
	clockScaffold = 0;
	clockAnnotations = 0;
	scaffoldInMemory = new HashMap<Integer, Pair<SortedSet<Integer>, SortedSet<Integer>>>();
	edgesInMemory = 0;
	hashToID = new HashMap<String, Integer>();
	idToHash = new HashMap<Integer, String>();
	alreadyRenamed = new Vector<String>();
	compresser = new Deflater(Deflater.BEST_COMPRESSION);
	W=10;
	L=5;
	nextVertexID = 0;
	try {
		benchmarks = new PrintWriter("/Users/melanie/Documents/benchmarks/compression_time_berkeleyDB.txt", "UTF-8");
		// Open the environment. Create it if it does not already exist.
		EnvironmentConfig envConfig = new EnvironmentConfig();
		envConfig.setAllowCreate(true);
		DatabaseEnvironment1 = new Environment(new File(filePath + "/scaffold"), 
				envConfig);
		DatabaseEnvironment2 = new Environment(new File(filePath + "/annotations"), 
				envConfig);
		// Open the databases. Create it if it does not already exist.
		DatabaseConfig DatabaseConfig1 = new DatabaseConfig();
		DatabaseConfig1.setAllowCreate(true);
		scaffoldDatabase = DatabaseEnvironment1.openDatabase(null, "spade_scaffold", DatabaseConfig1); 
		annotationsDatabase = DatabaseEnvironment2.openDatabase(null, "spade_annotations", DatabaseConfig1); 

		return true;

	} catch (Exception ex) {
		// Exception handling goes here
		logger.log(Level.SEVERE, "Compressed Storage Initialized not successful!", ex);
		return false;
	}
}
 

@Override
public void open() throws Exception {
    File dir = new File(crawlPath);
    if (!dir.exists()) {
        dir.mkdirs();
    }
    EnvironmentConfig environmentConfig = new EnvironmentConfig();
    environmentConfig.setAllowCreate(true);

    env = new Environment(dir, environmentConfig);
}
 

public TransactionBJEImpl(Transaction t, Environment env)
	{
		this.t = t;
		this.env = env;
//		if (t != null && traceme)
//			System.out.println("Created BJE tx " + t.getId());
	}
 

public BerkeleyDBReader(String crawlPath) {
    this.crawlPath = crawlPath;
    File dir = new File(crawlPath);
    EnvironmentConfig environmentConfig = new EnvironmentConfig();
    environmentConfig.setAllowCreate(true);
    env = new Environment(dir, environmentConfig);
    crawldbDatabase = env.openDatabase(null, "crawldb", BerkeleyDBUtils.defaultDBConfig);
    cursor = crawldbDatabase.openCursor(null, CursorConfig.DEFAULT);
}