Java Code Examples for com.sleepycat.je.Cursor#getNext()

private int getConfigVersion(Database configVersionDb)
{
    Cursor cursor = null;
    try
    {
        cursor = configVersionDb.openCursor(null, null);
        DatabaseEntry key = new DatabaseEntry();
        DatabaseEntry value = new DatabaseEntry();
        while (cursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS)
        {
            return IntegerBinding.entryToInt(value);
        }
        return -1;
    }
    finally
    {
        if (cursor != null)
        {
            cursor.close();
        }
    }
}
 

@VisibleForTesting
public void readData(int limit)
{
    Cursor cursor = scaffoldDatabase.openCursor(null, null);
    DatabaseEntry key = new DatabaseEntry();
    DatabaseEntry data = new DatabaseEntry();
    int i = 0;
    while(cursor.getNext(key, data, LockMode.DEFAULT) == OperationStatus.SUCCESS)
    {
        if(i >= limit)
            break;
        String keyString = new String(key.getData());
        System.out.println("hash: " + keyString);
        i++;
    }
    cursor.close();
}
 

private Set privateLoadAllKeys() {
    Set keys = new java.util.HashSet((int) _db.count());
    Cursor cursor = null;
    try {
        cursor = _db.openCursor(null, null);
        DatabaseEntry foundKey = new DatabaseEntry();
        DatabaseEntry foundData = new DatabaseEntry();

        while (cursor.getNext(foundKey, foundData, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
            keys.add((K) entryToObject(foundKey));
        }
    } catch (Exception e) {
        _logger.log(Level.SEVERE, e.getMessage(), e);
    } finally {
        cursor.close();
    }

    _logger.log(Level.INFO, this.getClass().getCanonicalName() + ":" + _mapName + ":loadAllKeys:" + keys.size());

    return keys;
}
 

public void list() throws Exception {
        if (env == null) {
            open();
        }
        Cursor cursor = null;
        Database crawldbDatabase = env.openDatabase(null, "crawldb", BerkeleyDBUtils.defaultDBConfig);
        cursor = crawldbDatabase.openCursor(null, CursorConfig.DEFAULT);
        DatabaseEntry key = new DatabaseEntry();
        DatabaseEntry value = new DatabaseEntry();

        while (cursor.getNext(key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
            CrawlDatum datum = BerkeleyDBUtils.createCrawlDatum(key, value);
            System.out.println(CrawlDatumFormater.datumToString(datum));
//            try {
//                CrawlDatum datum = BerkeleyDBUtils.createCrawlDatum(key, value);
//                System.out.println(CrawlDatumFormater.datumToString(datum));
//            } catch (Exception ex) {
//                LOG.info("Exception when generating", ex);
//                continue;
//            }
        }

    }
 

public List<String> dump(String prefix, int start, int count, LockMode lockMode) {
  EntryBinding<String> binder = TupleBinding.getPrimitiveBinding(String.class);
  DatabaseEntry key = new DatabaseEntry();
  binder.objectToEntry(prefix, key);
  DatabaseEntry value = new DatabaseEntry();

  Cursor cursor = database.openCursor(null, null);
  OperationStatus status = cursor.getSearchKeyRange(key, value, LockMode.READ_UNCOMMITTED);
  List<String> result = new ArrayList<>();
  int index = 0;
  while (status == OperationStatus.SUCCESS && index < (start + count)) {
    if (index >= start) {
      result.add(
        binder.entryToObject(key) + " -> " + binder.entryToObject(value)
      );
    }
    index++;
    status = cursor.getNext(key, value, LockMode.READ_UNCOMMITTED);
  }
  cursor.close();
  return result;
}
 

int getSourceVersion(Database versionDb)
{
    int version = -1;

    Cursor cursor = null;
    try
    {
        cursor = versionDb.openCursor(null, null);

        DatabaseEntry key = new DatabaseEntry();
        DatabaseEntry value = new DatabaseEntry();

        while(cursor.getNext(key, value, null) == OperationStatus.SUCCESS)
        {
            int ver = IntegerBinding.entryToInt(key);
            if(ver > version)
            {
                version = ver;
            }
        }
    }
    finally
    {
        if(cursor != null)
        {
            cursor.close();
        }
    }


    return version;
}
 

private long getMaximumMessageId(Database messageMetaDataDb)
{
    Cursor cursor = null;
    long maximumMessageId = 0;  // Our hand-rolled sequences value always began at zero
    try
    {
        cursor = messageMetaDataDb.openCursor(null, null);
        DatabaseEntry key = new DatabaseEntry();
        DatabaseEntry value = new DatabaseEntry();
        MessageMetaDataBinding valueBinding = MessageMetaDataBinding.getInstance();

        while (cursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS)
        {
            long messageId = LongBinding.entryToLong(key);
            maximumMessageId = Math.max(messageId, maximumMessageId);
        }
    }
    finally
    {
        if (cursor != null)
        {
            cursor.close();
        }
    }

    return maximumMessageId;
}
 

@Override
public void merge() throws Exception {
    LOG.info("start merge");
    Database crawldbDatabase = env.openDatabase(null, "crawldb", BerkeleyDBUtils.defaultDBConfig);
    /*合并fetch库*/
    LOG.info("merge fetch database");
    Database fetchDatabase = env.openDatabase(null, "fetch", BerkeleyDBUtils.defaultDBConfig);
    Cursor fetchCursor = fetchDatabase.openCursor(null, null);
    DatabaseEntry key = new DatabaseEntry();
    DatabaseEntry value = new DatabaseEntry();
    while (fetchCursor.getNext(key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
        crawldbDatabase.put(null, key, value);
    }
    fetchCursor.close();
    fetchDatabase.close();
    /*合并link库*/
    LOG.info("merge link database");
    Database linkDatabase = env.openDatabase(null, "link", BerkeleyDBUtils.defaultDBConfig);
    Cursor linkCursor = linkDatabase.openCursor(null, null);
    while (linkCursor.getNext(key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
        if (!(crawldbDatabase.get(null, key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS)) {
            crawldbDatabase.put(null, key, value);
        }
    }
    linkCursor.close();
    linkDatabase.close();
    LOG.info("end merge");
    crawldbDatabase.close();

    env.removeDatabase(null, "fetch");
    LOG.debug("remove fetch database");
    env.removeDatabase(null, "link");
    LOG.debug("remove link database");

}
 

/**
 * @return a (sorted) map of offset -> data for the given message id
 */
private SortedMap<Integer, byte[]> getMessageData(final long messageId, final Database oldDatabase)
{
    TreeMap<Integer, byte[]> data = new TreeMap<Integer, byte[]>();

    Cursor cursor = oldDatabase.openCursor(null, CursorConfig.READ_COMMITTED);
    try
    {
        DatabaseEntry contentKeyEntry = new DatabaseEntry();
        DatabaseEntry value = new DatabaseEntry();
        CompoundKeyBinding binding = new CompoundKeyBinding();
        binding.objectToEntry(new CompoundKey(messageId, 0), contentKeyEntry);

        OperationStatus status = cursor.getSearchKeyRange(contentKeyEntry, value, LockMode.DEFAULT);
        OldDataBinding dataBinding = new OldDataBinding();

        while (status == OperationStatus.SUCCESS)
        {
            CompoundKey compoundKey = binding.entryToObject(contentKeyEntry);
            long id = compoundKey.getMessageId();

            if (id != messageId)
            {
                // we have exhausted all chunks for this message id, break
                break;
            }

            int offsetInMessage = compoundKey.getOffset();
            OldDataValue dataValue = dataBinding.entryToObject(value);
            data.put(offsetInMessage, dataValue.getData());

            status = cursor.getNext(contentKeyEntry, value, LockMode.DEFAULT);
        }
    }
    finally
    {
        cursor.close();
    }

    return data;
}
 

/**
 * Split the region
 * @param sampleSize
 * @param numberOfElements
 * @return
 */
private void splitRegion(final Hyperrectangle boundingBoxToSplit) {

	final int parentBoxDimension = boxDimension.get(boundingBoxToSplit) % dataDimension;

	final double splitPosition = getSplitPosition(boundingBoxToSplit, parentBoxDimension);

	final Hyperrectangle leftBBox = boundingBoxToSplit.splitAndGetLeft(splitPosition,
			parentBoxDimension, true);

	final Hyperrectangle rightBBox = boundingBoxToSplit.splitAndGetRight(splitPosition,
			parentBoxDimension, false);

	// Write the box dimension
	boxDimension.put(leftBBox, parentBoxDimension + 1);
	boxDimension.put(rightBBox, parentBoxDimension + 1);

	// Insert new boxes and remove old one
	elements.put(leftBBox, buildNewDatabase());
	elements.put(rightBBox, buildNewDatabase());

	final Database database = elements.remove(boundingBoxToSplit);

	// Data to redistribute
    final Cursor cursor = database.openCursor(null, null);

    final DatabaseEntry foundKey = new DatabaseEntry();
    final DatabaseEntry foundData = new DatabaseEntry();

    while(cursor.getNext(foundKey, foundData, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
        final Hyperrectangle box = Hyperrectangle.fromByteArray(foundData.getData());

        boolean redistributed = false;

        if(leftBBox.intersects(box)) {
			elements.get(leftBBox).put(null, foundKey, foundData);
			elementCounter.computeIfAbsent(leftBBox, l -> new AtomicLong(0)).incrementAndGet();
			redistributed = true;
		}

		if(rightBBox.intersects(box)) {
			elements.get(rightBBox).put(null, foundKey, foundData);
			elementCounter.computeIfAbsent(rightBBox, l -> new AtomicLong(0)).incrementAndGet();
			redistributed = true;
		}

		if(! redistributed) {
			System.err.println("Unable to redistribute: " + box + " left / right "
					+ leftBBox + " " + rightBBox);
		}
    }


    cursor.close();

    // Name needs to be fetched before database is closed
    final String databaseName = database.getDatabaseName();

    database.close();

	dbEnv.removeDatabase(null, databaseName);
}