com.sleepycat.je.Database Java Examples

/**
 * Constructor
 * @param name The unique name of this manager.
 * @param shortName Original name for the Manager.
 * @param agents An array of Agents.
 * @param batchSize The number of events to include in a batch.
 * @param retries The number of times to retry connecting before giving up.
 * @param connectionTimeout The amount of time to wait for a connection to be established.
 * @param requestTimeout The amount of time to wair for a response to a request.
 * @param delay The amount of time to wait between retries.
 * @param database The database to write to.
 * @param environment The database environment.
 * @param secretKey The SecretKey to use for encryption.
 * @param lockTimeoutRetryCount The number of times to retry a lock timeout.
 */
protected FlumePersistentManager(final String name, final String shortName, final Agent[] agents,
                                 final int batchSize, final int retries, final int connectionTimeout,
                                 final int requestTimeout, final int delay, final Database database,
                                 final Environment environment, final SecretKey secretKey,
                                 final int lockTimeoutRetryCount) {
    super(name, shortName, agents, batchSize, delay, retries, connectionTimeout, requestTimeout);
    this.database = database;
    this.environment = environment;
    dbCount.set(database.count());
    this.worker = new WriterThread(database, environment, this, gate, batchSize, secretKey, dbCount,
        lockTimeoutRetryCount);
    this.worker.start();
    this.secretKey = secretKey;
    this.threadPool = Executors.newCachedThreadPool(Log4jThreadFactory.createDaemonThreadFactory("Flume"));
    this.lockTimeoutRetryCount = lockTimeoutRetryCount;
}
 

public Database getDatabase(String name) {
    Database db = databases.get(name);
    if (db == null) {
        synchronized (this) {
            db = databases.get(name);

            if (db == null) {
                db = ((JE_Repository) this.repo).getDatabase(name, schema.isTmp(), schema.issortedDuplicates());
                databases.put(name, db);
            }

        }
    }

    return db;
}
 

private Database getLinksVersionDb()
{
    Database linksVersionDb;
    try
    {
        DatabaseConfig config = new DatabaseConfig().setTransactional(true).setAllowCreate(false);
        linksVersionDb = getEnvironmentFacade().openDatabase(LINKS_VERSION_DB_NAME, config);
    }
    catch (DatabaseNotFoundException e)
    {
        updateVersion(null, BrokerModel.MODEL_VERSION);
        linksVersionDb = getEnvironmentFacade().openDatabase(LINKS_VERSION_DB_NAME, DEFAULT_DATABASE_CONFIG);
    }

    return linksVersionDb;
}
 

private void storeConfiguredObjectEntry(Database configuredObjectsDb, final Transaction txn, ConfiguredObjectRecord configuredObject)
{
    DatabaseEntry key = new DatabaseEntry();
    UUIDTupleBinding uuidBinding = UUIDTupleBinding.getInstance();
    uuidBinding.objectToEntry(configuredObject.getId(), key);

    DatabaseEntry value = new DatabaseEntry();
    ConfiguredObjectBinding configuredObjectBinding = ConfiguredObjectBinding.getInstance();

    configuredObjectBinding.objectToEntry(configuredObject, value);
    OperationStatus status = configuredObjectsDb.put(txn, key, value);
    if (status != OperationStatus.SUCCESS)
    {
        throw new StoreException("Error writing configured object " + configuredObject + " to database: "
                + status);
    }
}
 

@Override
public <KeyT, ValueT> BdbWrapper<KeyT, ValueT> getDatabase(String userId, String dataSetId, String databaseName,
                                                           boolean allowDuplicates, EntryBinding<KeyT> keyBinder,
                                                           EntryBinding<ValueT> valueBinder,
                                                           IsCleanHandler<KeyT, ValueT> isCleanHandler)
  throws BdbDbCreationException {
  try {
    DatabaseConfig config = new DatabaseConfig();
    config.setAllowCreate(true);
    config.setDeferredWrite(true);
    config.setSortedDuplicates(allowDuplicates);

    String environmentKey = environmentKey(userId, dataSetId);
    File envHome = new File(dbHome, environmentKey);
    envHome.mkdirs();
    Environment dataSetEnvironment = new Environment(envHome, configuration);
    Database database = dataSetEnvironment.openDatabase(null, databaseName, config);
    databases.put(environmentKey + "_" + databaseName, database);
    environmentMap.put(environmentKey, dataSetEnvironment);
    return new BdbWrapper<>(dataSetEnvironment, database, config, keyBinder, valueBinder, isCleanHandler);
  } catch (DatabaseException e) {
    throw new BdbDbCreationException(e);
  }
}
 

@Override
public void run(final Database sourceDatabase, final Database targetDatabase, final Transaction transaction)
{
    _rowCount = sourceDatabase.count();
    _template = new CursorTemplate(sourceDatabase, transaction, new DatabaseEntryCallback()
    {
        @Override
        public void processEntry(final Database database, final Transaction transaction, final DatabaseEntry key,
                final DatabaseEntry value)
        {
            _processedRowCount++;
            CursorOperation.this.processEntry(database, targetDatabase, transaction, key, value);
            if (getProcessedCount() % 1000 == 0)
            {
                LOGGER.info("Processed " + getProcessedCount() + " records of " + getRowCount() + ".");
            }
        }

    });
    _template.processEntries();
}
 

private List<AMQShortString> findTopicExchanges(final Environment environment)
{
    final List<AMQShortString> topicExchanges = new ArrayList<AMQShortString>();
    final ExchangeRecordBinding binding = new ExchangeRecordBinding();
    CursorOperation databaseOperation = new CursorOperation()
    {

        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction,
                DatabaseEntry key, DatabaseEntry value)
        {
            ExchangeRecord record = binding.entryToObject(value);
            if (AMQShortString.valueOf(ExchangeDefaults.TOPIC_EXCHANGE_CLASS).equals(record.getType()))
            {
                topicExchanges.add(record.getName());
            }
        }
    };
    new DatabaseTemplate(environment, EXCHANGE_DB_NAME, null).run(databaseOperation);
    return topicExchanges;
}
 

@Override
public Database openDatabase(String name, DatabaseConfig databaseConfig)
{
    Database cachedHandle = _cachedDatabases.get(name);
    if (cachedHandle == null)
    {
        Database handle = getEnvironment().openDatabase(null, name, databaseConfig);
        Database existingHandle = _cachedDatabases.putIfAbsent(name, handle);
        if (existingHandle == null)
        {
            cachedHandle = handle;
        }
        else
        {
            cachedHandle = existingHandle;
            handle.close();
        }
    }
    return cachedHandle;
}
 

@Override
public Sequence openSequence(final Database database,
                             final DatabaseEntry sequenceKey,
                             final SequenceConfig sequenceConfig)
{
    Sequence cachedSequence = _cachedSequences.get(sequenceKey);
    if (cachedSequence == null)
    {
        Sequence handle = database.openSequence(null, sequenceKey, sequenceConfig);
        Sequence existingHandle = _cachedSequences.putIfAbsent(sequenceKey, handle);
        if (existingHandle == null)
        {
            cachedSequence = handle;
        }
        else
        {
            cachedSequence = existingHandle;
            handle.close();
        }
    }
    return cachedSequence;
}
 

public void close() throws DatabaseException, IOException {
  for (Database database : databases.values()) {
    database.close();
  }
  boolean wasDeleted = false;
  int tries = 0;
  while (!wasDeleted) {
    try {
      FileUtils.cleanDirectory(dbHome);
      wasDeleted = true;
    } catch (IOException e) {
      tries++;
      if (tries >= 10) {
        wasDeleted = true;
      } else {
        try {
          Thread.sleep(1);
        } catch (InterruptedException e1) {
          LOG.error("Trying to clean up and delete directory, but it failed the first time around and then the " +
            "thread was interrupted");
          wasDeleted = true;
        }
      }
    }
  }
}
 

@Test
public void testOpenDatabaseReusesCachedHandle() throws Exception
{
    DatabaseConfig createIfAbsentDbConfig = DatabaseConfig.DEFAULT.setAllowCreate(true);

    EnvironmentFacade ef = createMaster();
    Database handle1 = ef.openDatabase("myDatabase", createIfAbsentDbConfig);
    assertNotNull(handle1);

    Database handle2 = ef.openDatabase("myDatabase", createIfAbsentDbConfig);
    assertSame("Database handle should be cached", handle1, handle2);

    ef.closeDatabase("myDatabase");

    Database handle3 = ef.openDatabase("myDatabase", createIfAbsentDbConfig);
    assertNotSame("Expecting a new handle after database closure", handle1, handle3);
}
 

private List<BindingRecord> loadBindings()
{
    final BindingTuple bindingTuple = new BindingTuple();
    final List<BindingRecord> queueBindings = new ArrayList<BindingRecord>();
    CursorOperation databaseOperation = new CursorOperation()
    {

        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction,
                DatabaseEntry key, DatabaseEntry value)
        {
            BindingRecord bindingRecord = bindingTuple.entryToObject(key);

            AMQShortString queueName = bindingRecord.getQueueName();
            AMQShortString exchangeName = bindingRecord.getExchangeName();
            AMQShortString routingKey = bindingRecord.getRoutingKey();
            FieldTable arguments = bindingRecord.getArguments();
            queueBindings.add(new BindingRecord(exchangeName, queueName, routingKey, arguments));
        }
    };
    new DatabaseTemplate(_environment, BINDING_DB_NAME, null).run(databaseOperation);
    return queueBindings;
}
 

private void assertContent()
{
    final UpgradeFrom4To5.ContentBinding contentBinding = new UpgradeFrom4To5.ContentBinding();
    CursorOperation contentCursorOperation = new CursorOperation()
    {

        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction, DatabaseEntry key,
                DatabaseEntry value)
        {
            long id = LongBinding.entryToLong(key);
            assertTrue("Unexpected id", id > 0);
            ByteBuffer content = contentBinding.entryToObject(value);
            assertNotNull("Unexpected content", content);
        }
    };
    new DatabaseTemplate(_environment, MESSAGE_CONTENT_DB_NAME, null).run(contentCursorOperation);
}
 

private void assertQueueEntries()
{
    final Map<UUID, UpgradeConfiguredObjectRecord> configuredObjects = loadConfiguredObjects();
    final NewQueueEntryBinding newBinding = new NewQueueEntryBinding();
    CursorOperation cursorOperation = new CursorOperation()
    {

        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction,
                DatabaseEntry key, DatabaseEntry value)
        {
            NewQueueEntryKey newEntryRecord = newBinding.entryToObject(key);
            assertTrue("Unexpected queue id", configuredObjects.containsKey(newEntryRecord.getQueueId()));
        }
    };
    new DatabaseTemplate(_environment, NEW_DELIVERY_DB_NAME, null).run(cursorOperation);
}
 

private Map<UUID, UpgradeConfiguredObjectRecord> loadConfiguredObjects()
{
    final Map<UUID, UpgradeConfiguredObjectRecord> configuredObjectsRecords = new HashMap<UUID, UpgradeConfiguredObjectRecord>();
    final ConfiguredObjectBinding binding = new ConfiguredObjectBinding();
    final UpgradeUUIDBinding uuidBinding = new UpgradeUUIDBinding();
    CursorOperation configuredObjectsCursor = new CursorOperation()
    {
        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction,
                DatabaseEntry key, DatabaseEntry value)
        {
            UUID id = uuidBinding.entryToObject(key);
            UpgradeConfiguredObjectRecord object = binding.entryToObject(value);
            configuredObjectsRecords.put(id, object);
        }
    };
    new DatabaseTemplate(_environment, CONFIGURED_OBJECTS_DB_NAME, null).run(configuredObjectsCursor);
    return configuredObjectsRecords;
}
 

@Test
public void testExecuteWithTwoDatabases()
{
    String targetDatabaseName = "targetDatabase";
    Database targetDatabase = mock(Database.class);

    Transaction txn = mock(Transaction.class);

    when(_environment.openDatabase(same(txn), same(targetDatabaseName), isA(DatabaseConfig.class)))
            .thenReturn(targetDatabase);

    DatabaseTemplate databaseTemplate = new DatabaseTemplate(_environment, SOURCE_DATABASE, targetDatabaseName, txn);

    DatabaseRunnable databaseOperation = mock(DatabaseRunnable.class);
    databaseTemplate.run(databaseOperation);

    verify(databaseOperation).run(_sourceDatabase, targetDatabase, txn);
    verify(_sourceDatabase).close();
    verify(targetDatabase).close();
}
 

public Database getDatabase(String name, boolean isTmp, boolean isSortedDuplicates) {
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setAllowCreate(true);
    Environment _env = env;
    if (isTmp) {
        dbConfig.setTemporary(true);
        dbConfig.setSortedDuplicates(isSortedDuplicates);
        _env = getTmpEnv();
    } else {
        if (!config.isTransactional()) {
            dbConfig.setDeferredWrite(config.isCommitSync());
        } else {
            dbConfig.setTransactional(true);
        }
    }

    Database database = buildPrimaryIndex(dbConfig, _env, name);
    return database;
}
 

private int getStoreVersion(Environment environment)
{
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setTransactional(true);
    dbConfig.setAllowCreate(true);
    int storeVersion = -1;
    try(Database versionDb = environment.openDatabase(null, Upgrader.VERSION_DB_NAME, dbConfig);
        Cursor cursor = versionDb.openCursor(null, null))
    {
        DatabaseEntry key = new DatabaseEntry();
        DatabaseEntry value = new DatabaseEntry();
        while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS)
        {
            int version = IntegerBinding.entryToInt(key);
            if (storeVersion < version)
            {
                storeVersion = version;
            }
        }
    }
    return storeVersion;
}
 

private void assertContent()
{
    final ByteBufferBinding contentBinding = ByteBufferBinding.getInstance();
    CursorOperation contentCursorOperation = new CursorOperation()
    {

        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction, DatabaseEntry key,
                DatabaseEntry value)
        {
            long id = LongBinding.entryToLong(key);
            assertTrue("Unexpected id", id > 0);
            QpidByteBuffer content = contentBinding.entryToObject(value);
            assertNotNull("Unexpected content", content);
            assertTrue("Expected content", content.hasRemaining());
        }
    };
    new DatabaseTemplate(_environment, "MESSAGE_CONTENT", null).run(contentCursorOperation);
}
 

private List<String> loadVersions()
{
    final List<String> versions = new ArrayList<>();
    CursorOperation configuredObjectsCursor = new CursorOperation()
    {
        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction,
                                 DatabaseEntry key, DatabaseEntry value)
        {
            String version = StringBinding.entryToString(key);
            versions.add(version);
        }
    };
    new DatabaseTemplate(_environment, PREFERENCES_VERSION_DB_NAME, null).run(configuredObjectsCursor);
    return versions;
}
 

private void assertXidEntries(Environment environment)
{
    final DatabaseEntry value = new DatabaseEntry();
    final DatabaseEntry key = getXidKey();
    new DatabaseTemplate(environment, NEW_XID_DB_NAME, null).run(new DatabaseRunnable()
    {

        @Override
        public void run(Database xidDatabase, Database nullDatabase, Transaction transaction)
        {
            xidDatabase.get(null, key, value, LockMode.DEFAULT);
        }
    });
    NewPreparedTransactionBinding newBinding = new NewPreparedTransactionBinding();
    NewPreparedTransaction newTransaction = newBinding.entryToObject(value);
    NewRecordImpl[] newEnqueues = newTransaction.getEnqueues();
    NewRecordImpl[] newDequeues = newTransaction.getDequeues();
    assertEquals("Unxpected new enqueus number", 1, newEnqueues.length);
    NewRecordImpl enqueue = newEnqueues[0];
    assertEquals("Unxpected queue id", UUIDGenerator.generateQueueUUID("TEST1", getVirtualHost().getName()), enqueue.getId());
    assertEquals("Unxpected message id", 1, enqueue.getMessageNumber());
    assertEquals("Unxpected new dequeues number", 1, newDequeues.length);
    NewRecordImpl dequeue = newDequeues[0];
    assertEquals("Unxpected queue id", UUIDGenerator.generateQueueUUID("TEST2", getVirtualHost().getName()), dequeue.getId());
    assertEquals("Unxpected message id", 2, dequeue.getMessageNumber());
}
 

@Override
protected void doDeleteLink(final LinkDefinition<Source, Target> linkDefinition)
{
    LinkKey linkKey = new LinkKey(linkDefinition);
    try
    {
        Database linksDatabase = getEnvironmentFacade().openDatabase(LINKS_DB_NAME, DEFAULT_DATABASE_CONFIG);

        final DatabaseEntry databaseEntry = new DatabaseEntry();
        LinkKeyEntryBinding.getInstance().objectToEntry(linkKey, databaseEntry);
        OperationStatus status = linksDatabase.delete(null, databaseEntry);
        if (status != OperationStatus.SUCCESS)
        {
            LOGGER.debug(String.format("Unexpected status '%s' for deletion of '%s'", status, linkKey));
        }
    }
    catch (RuntimeException e)
    {
        throw getEnvironmentFacade().handleDatabaseException(String.format("Failed deletion of link '%s'", linkKey), e);
    }
}
 

@Test
public void testOpenDatabaseReusesCachedHandle() throws Exception
{
    DatabaseConfig createIfAbsentDbConfig = DatabaseConfig.DEFAULT.setAllowCreate(true);

    EnvironmentFacade ef = createEnvironmentFacade();
    Database handle1 = ef.openDatabase("myDatabase", createIfAbsentDbConfig);
    assertNotNull(handle1);

    Database handle2 = ef.openDatabase("myDatabase", createIfAbsentDbConfig);
    assertSame("Database handle should be cached", handle1, handle2);

    ef.closeDatabase("myDatabase");

    Database handle3 = ef.openDatabase("myDatabase", createIfAbsentDbConfig);
    assertNotSame("Expecting a new handle after database closure", handle1, handle3);
}
 

private Map<UpgradeHierarchyKey, UUID> loadConfiguredObjectHierarchy()
{
    final Map<UpgradeHierarchyKey, UUID> hierarchyRecords = new HashMap<UpgradeHierarchyKey, UUID>();
    final UpgradeHierarchyKeyBinding hierarchyKeyBinding = new UpgradeHierarchyKeyBinding();
    final UpgradeUUIDBinding uuidParentBinding = new UpgradeUUIDBinding();
    CursorOperation hierarchyCursor = new CursorOperation()
    {
        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction,
                DatabaseEntry key, DatabaseEntry value)
        {
            UpgradeHierarchyKey hierarchyKey = hierarchyKeyBinding.entryToObject(key);
            UUID parentId = uuidParentBinding.entryToObject(value);
            hierarchyRecords.put(hierarchyKey, parentId);
        }
    };
    new DatabaseTemplate(_environment, CONFIGURED_OBJECT_HIERARCHY_DB_NAME, null).run(hierarchyCursor);
    return hierarchyRecords;
}
 

private Map<UUID, UpgradeConfiguredObjectRecord> loadConfiguredObjects()
{
    final Map<UUID, UpgradeConfiguredObjectRecord> configuredObjectsRecords = new HashMap<UUID, UpgradeConfiguredObjectRecord>();
    final UpgradeConfiguredObjectBinding binding = new UpgradeConfiguredObjectBinding();
    final UpgradeUUIDBinding uuidBinding = new UpgradeUUIDBinding();
    CursorOperation configuredObjectsCursor = new CursorOperation()
    {
        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction,
                DatabaseEntry key, DatabaseEntry value)
        {
            UUID id = uuidBinding.entryToObject(key);
            UpgradeConfiguredObjectRecord object = binding.entryToObject(value);
            configuredObjectsRecords.put(id, object);
        }
    };
    new DatabaseTemplate(_environment, CONFIGURED_OBJECTS_DB_NAME, null).run(configuredObjectsCursor);
    return configuredObjectsRecords;
}
 

public DatabaseGetterBuilderImpl(EntryBinding<KeyT> keyBinder, EntryBinding<ValueT> valueBinder, Database database,
                                 Map<Cursor, String> cursors, KeyT keyToIgnore, ValueT valueToIgnore) {
  this.keyBinder = keyBinder;
  this.valueBinder = valueBinder;
  this.database = database;
  this.cursors = cursors;
  this.keyToIgnore = keyToIgnore;
  this.valueToIgnore = valueToIgnore;
}
 

DatabaseGetter(EntryBinding<KeyT> keyBinder, EntryBinding<ValueT> valueBinder, DatabaseFunction initializer,
               DatabaseFunction iterator, Database database, Map<Cursor, String> cursors, DatabaseEntry key,
               DatabaseEntry value) {
  this.keyBinder = keyBinder;
  this.valueBinder = valueBinder;
  this.initializer = initializer;
  this.iterator = iterator;
  this.database = database;
  this.cursors = cursors;
  this.key = key;
  this.value = value;
}
 

private Set<Long> assertDeliveriesForQueue(final String queueName, final int expectedQueueSize)
{
    final QueueEntryKeyBinding queueEntryKeyBinding = new QueueEntryKeyBinding();
    final AtomicInteger deliveryCounter = new AtomicInteger();
    final Set<Long> messagesForQueue = new HashSet<Long>();

    CursorOperation deliveryDatabaseOperation = new CursorOperation()
    {
        @Override
        public void processEntry(Database sourceDatabase, Database targetDatabase, Transaction transaction,
                DatabaseEntry key, DatabaseEntry value)
        {
            QueueEntryKey entryKey = queueEntryKeyBinding.entryToObject(key);
            String thisQueueName = entryKey.getQueueName().toString();
            if (thisQueueName.equals(queueName))
            {
                deliveryCounter.incrementAndGet();
                messagesForQueue.add(entryKey.getMessageId());
            }
        }
    };
    new DatabaseTemplate(_environment, DELIVERY_DB_NAME, null).run(deliveryDatabaseOperation);

    assertEquals("Unxpected number of entries in delivery db for queue " + queueName, expectedQueueSize,
            deliveryCounter.get());

    return messagesForQueue;
}
 

public BDBWriter(final byte[] keyData, final byte[] eventData, final Environment environment,
                 final Database database, final Gate gate, final AtomicLong dbCount, final long batchSize,
                 final int lockTimeoutRetryCount) {
    this.keyData = keyData;
    this.eventData = eventData;
    this.environment = environment;
    this.database = database;
    this.gate = gate;
    this.dbCount = dbCount;
    this.batchSize = batchSize;
    this.lockTimeoutRetryCount = lockTimeoutRetryCount;
}
 

/**
 * Init a new BDB environment in the given folder
 * @param folder
 */
@SuppressWarnings("unused")
protected void initNewBDBEnvironment(final File folder, final EnvironmentConfig envConfig) {

	final Environment dbEnv = new Environment(folder, envConfig);

	Transaction txn = null;
	if(USE_TRANSACTIONS) {
		txn = dbEnv.beginTransaction(null, null);
	}
	
	final DatabaseConfig dbConfig = new DatabaseConfig();
	dbConfig.setTransactional(USE_TRANSACTIONS);
	dbConfig.setAllowCreate(true);
	dbConfig.setSortedDuplicates(true);
	dbConfig.setDeferredWrite(true);
	//dbConfig.setKeyPrefixing(true);
	//dbConfig.setNodeMaxEntries(128);

	final Database database = dbEnv.openDatabase(txn, "osm", dbConfig);

	if(txn != null) {
		txn.commit();
	}
	
	environments.add(dbEnv);
	databases.add(database);
}