Java Code Examples for org.iq80.leveldb.Options#createIfMissing()

The following examples show how to use org.iq80.leveldb.Options#createIfMissing() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ShuffleHandler.java    From tez with Apache License 2.0 6 votes vote down vote up 
private void startStore(Path recoveryRoot) throws IOException {
  Options options = new Options();
  options.createIfMissing(false);
  options.logger(new LevelDBLogger());
  Path dbPath = new Path(recoveryRoot, STATE_DB_NAME);
  LOG.info("Using state database at " + dbPath + " for recovery");
  File dbfile = new File(dbPath.toString());
  try {
    stateDb = JniDBFactory.factory.open(dbfile, options);
  } catch (NativeDB.DBException e) {
    if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
      LOG.info("Creating state database at " + dbfile);
      options.createIfMissing(true);
      try {
        stateDb = JniDBFactory.factory.open(dbfile, options);
        storeVersion();
      } catch (DBException dbExc) {
        throw new IOException("Unable to create state store", dbExc);
      }
    } else {
      throw e;
    }
  }
  checkVersion();
}
Example 2
Source File: LevelDB.java    From SPADE with GNU General Public License v3.0 6 votes vote down vote up 
public boolean initialize(String arguments)
    {
        try
        {
            WriteOptions writeOptions = new WriteOptions();
            writeOptions.sync(false);

            directoryPath = arguments;
            Options options = new Options();
            options.createIfMissing(true);
            options.compressionType(CompressionType.NONE);
//            scaffoldDatabase = factory.open(new File(directoryPath), options);
            childDatabase = factory.open(new File(directoryPath + "child"), options);
            parentDatabase = factory.open(new File(directoryPath + "parent"), options);
            logger.log(Level.INFO, "Scaffold initialized");
//            globalTxCheckin(true);
        }
        catch(IOException ex)
        {
            logger.log(Level.SEVERE, null, ex);
            return false;
        }

        return true;
    }
Example 3
Source File: EzLevelDbTable.java    From ezdb with Apache License 2.0 6 votes vote down vote up 
public EzLevelDbTable(File path, EzLevelDbFactory factory,
		Serde<H> hashKeySerde, Serde<R> rangeKeySerde, Serde<V> valueSerde,
		Comparator<byte[]> hashKeyComparator,
		Comparator<byte[]> rangeKeyComparator) {
	this.hashKeySerde = hashKeySerde;
	this.rangeKeySerde = rangeKeySerde;
	this.valueSerde = valueSerde;
	this.hashKeyComparator = hashKeyComparator;
	this.rangeKeyComparator = rangeKeyComparator;

	Options options = new Options();
	options.createIfMissing(true);
	options.comparator(new EzLevelDbComparator(hashKeyComparator,
			rangeKeyComparator));

	try {
		this.db = factory.open(path, options);
	} catch (IOException e) {
		throw new DbException(e);
	}
}
Example 4
Source File: HistoryServerLeveldbStateStoreService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
protected void startStorage() throws IOException {
  Path storeRoot = createStorageDir(getConfig());
  Options options = new Options();
  options.createIfMissing(false);
  options.logger(new LeveldbLogger());
  LOG.info("Using state database at " + storeRoot + " for recovery");
  File dbfile = new File(storeRoot.toString());
  try {
    db = JniDBFactory.factory.open(dbfile, options);
  } catch (NativeDB.DBException e) {
    if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
      LOG.info("Creating state database at " + dbfile);
      options.createIfMissing(true);
      try {
        db = JniDBFactory.factory.open(dbfile, options);
        // store version
        storeVersion();
      } catch (DBException dbErr) {
        throw new IOException(dbErr.getMessage(), dbErr);
      }
    } else {
      throw e;
    }
  }
  checkVersion();
}
Example 5
Source File: WarpRepair.java    From warp10-platform with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws IOException {
  String path = args[0];

  Options options = new Options();
  options.createIfMissing(false);
  options.maxOpenFiles(200);
  options.verifyChecksums(!("true".equals(System.getProperty(DISABLE_CHECKSUMS))));
  options.paranoidChecks(!("true".equals(System.getProperty(DISABLE_PARANOIDCHECKS))));
  
  boolean nativedisabled = "true".equals(System.getProperty(Configuration.LEVELDB_NATIVE_DISABLE));
  boolean javadisabled = "true".equals(System.getProperty(Configuration.LEVELDB_JAVA_DISABLE));

  repair(path, options, javadisabled, nativedisabled);
}
Example 6
Source File: PBImageTextWriter.java    From big-c with Apache License 2.0 5 votes vote down vote up 
LevelDBStore(final File dbPath) throws IOException {
  Options options = new Options();
  options.createIfMissing(true);
  options.errorIfExists(true);
  db = JniDBFactory.factory.open(dbPath, options);
  batch = db.createWriteBatch();
}
Example 7
Source File: HistoryServerLeveldbStateStoreService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
protected void startStorage() throws IOException {
  Path storeRoot = createStorageDir(getConfig());
  Options options = new Options();
  options.createIfMissing(false);
  options.logger(new LeveldbLogger());
  LOG.info("Using state database at " + storeRoot + " for recovery");
  File dbfile = new File(storeRoot.toString());
  try {
    db = JniDBFactory.factory.open(dbfile, options);
  } catch (NativeDB.DBException e) {
    if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
      LOG.info("Creating state database at " + dbfile);
      options.createIfMissing(true);
      try {
        db = JniDBFactory.factory.open(dbfile, options);
        // store version
        storeVersion();
      } catch (DBException dbErr) {
        throw new IOException(dbErr.getMessage(), dbErr);
      }
    } else {
      throw e;
    }
  }
  checkVersion();
}
Example 8
Source File: NMLeveldbStateStoreService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
protected void initStorage(Configuration conf)
    throws IOException {
  Path storeRoot = createStorageDir(conf);
  Options options = new Options();
  options.createIfMissing(false);
  options.logger(new LeveldbLogger());
  LOG.info("Using state database at " + storeRoot + " for recovery");
  File dbfile = new File(storeRoot.toString());
  try {
    db = JniDBFactory.factory.open(dbfile, options);
  } catch (NativeDB.DBException e) {
    if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
      LOG.info("Creating state database at " + dbfile);
      isNewlyCreated = true;
      options.createIfMissing(true);
      try {
        db = JniDBFactory.factory.open(dbfile, options);
        // store version
        storeVersion();
      } catch (DBException dbErr) {
        throw new IOException(dbErr.getMessage(), dbErr);
      }
    } else {
      throw e;
    }
  }
  checkVersion();
}
Example 9
Source File: WarpInit.java    From warp10-platform with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws IOException {
  String path = args[0];
  
  Options options = new Options();
  options.createIfMissing(true);
  options.verifyChecksums(true);
  options.paranoidChecks(true);

  DB db = null;

  boolean nativedisabled = "true".equals(System.getProperty(Configuration.LEVELDB_NATIVE_DISABLE));
  boolean javadisabled = "true".equals(System.getProperty(Configuration.LEVELDB_JAVA_DISABLE));
  
  try {
    if (!nativedisabled) {
      db = JniDBFactory.factory.open(new File(path), options);
    } else {
      throw new UnsatisfiedLinkError("Native LevelDB implementation disabled.");
    }
  } catch (UnsatisfiedLinkError ule) {
    ule.printStackTrace();
    if (!javadisabled) {
      db = Iq80DBFactory.factory.open(new File(path), options);
    } else {
      throw new RuntimeException("No usable LevelDB implementation, aborting.");
    }
  }      
  
  db.close();
}
Example 10
Source File: PBImageTextWriter.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
LevelDBStore(final File dbPath) throws IOException {
  Options options = new Options();
  options.createIfMissing(true);
  options.errorIfExists(true);
  db = JniDBFactory.factory.open(dbPath, options);
  batch = db.createWriteBatch();
}
Example 11
Source File: JLevelDBState.java    From jesos with Apache License 2.0 5 votes vote down vote up 
public JLevelDBState(final String path)
                throws IOException
{
    checkNotNull(path, "path is null");
    final Options options = new Options();
    options.createIfMissing(true);
    this.db = Iq80DBFactory.factory.open(new File(path), options);
}
Example 12
Source File: NMLeveldbStateStoreService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
protected void initStorage(Configuration conf)
    throws IOException {
  Path storeRoot = createStorageDir(conf);
  Options options = new Options();
  options.createIfMissing(false);
  options.logger(new LeveldbLogger());
  LOG.info("Using state database at " + storeRoot + " for recovery");
  File dbfile = new File(storeRoot.toString());
  try {
    db = JniDBFactory.factory.open(dbfile, options);
  } catch (NativeDB.DBException e) {
    if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
      LOG.info("Creating state database at " + dbfile);
      isNewlyCreated = true;
      options.createIfMissing(true);
      try {
        db = JniDBFactory.factory.open(dbfile, options);
        // store version
        storeVersion();
      } catch (DBException dbErr) {
        throw new IOException(dbErr.getMessage(), dbErr);
      }
    } else {
      throw e;
    }
  }
  checkVersion();
}
Example 13
Source File: LevelDBJobStore.java    From AthenaX with Apache License 2.0 5 votes vote down vote up 
@Override
public void open(AthenaXConfiguration conf) throws IOException {
  String dbLocation = (String) conf.extras().get(AthenaXExtraConfigOptions.JOBSTORE_LEVELDB_FILE.key());
  Preconditions.checkNotNull(dbLocation, "Invalid location of the db");
  Options options = new Options();
  options.createIfMissing(true);
  db = factory.open(new File(dbLocation), options);
}
Example 14
Source File: Storage.java    From gsc-core with GNU Lesser General Public License v3.0 5 votes vote down vote up 
private static Options createDefaultDbOptions() {
    Options dbOptions = new Options();

    dbOptions.createIfMissing(true);
    dbOptions.paranoidChecks(true);
    dbOptions.verifyChecksums(true);

    dbOptions.compressionType(DEFAULT_COMPRESSION_TYPE);
    dbOptions.blockSize(DEFAULT_BLOCK_SIZE);
    dbOptions.writeBufferSize(DEFAULT_WRITE_BUFFER_SIZE);
    dbOptions.cacheSize(DEFAULT_CACHE_SIZE);
    dbOptions.maxOpenFiles(DEFAULT_MAX_OPEN_FILES);

    return dbOptions;
}
Example 15
Source File: LevelDbDataSourceImpl.java    From gsc-core with GNU Lesser General Public License v3.0 5 votes vote down vote up 
@Deprecated
private Options createDbOptions() {
    Options dbOptions = new Options();
    dbOptions.createIfMissing(true);
    dbOptions.compressionType(CompressionType.NONE);
    dbOptions.blockSize(10 * 1024 * 1024);
    dbOptions.writeBufferSize(10 * 1024 * 1024);
    dbOptions.cacheSize(0);
    dbOptions.paranoidChecks(true);
    dbOptions.verifyChecksums(true);
    dbOptions.maxOpenFiles(32);
    return dbOptions;
}
Example 16
Source File: LevelDBStore.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up 
public LevelDBStore(File dbPath, boolean createIfMissing)
    throws IOException {
  dbOptions = new Options();
  dbOptions.createIfMissing(createIfMissing);
  this.dbFile = dbPath;
  this.writeOptions = new WriteOptions().sync(true);
  openDB(dbPath, dbOptions);
}
Example 17
Source File: MetadataStoreBuilder.java    From hadoop-ozone with Apache License 2.0 4 votes vote down vote up 
public MetadataStore build() throws IOException {
  if (dbFile == null) {
    throw new IllegalArgumentException("Failed to build metadata store, "
        + "dbFile is required but not found");
  }

  // Build db store based on configuration
  final ConfigurationSource conf = optionalConf.orElse(DEFAULT_CONF);

  if (dbType == null) {
    LOG.debug("dbType is null, using ");
    dbType = conf.getTrimmed(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
            OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT);
    LOG.debug("dbType is null, using dbType {} from ozone configuration",
        dbType);
  } else {
    LOG.debug("Using dbType {} for metastore", dbType);
  }
  if (OZONE_METADATA_STORE_IMPL_LEVELDB.equals(dbType)) {
    Options options = new Options();
    options.createIfMissing(createIfMissing);
    if (cacheSize > 0) {
      options.cacheSize(cacheSize);
    }
    return new LevelDBStore(dbFile, options);
  } else if (OZONE_METADATA_STORE_IMPL_ROCKSDB.equals(dbType)) {
    org.rocksdb.Options opts;
    // Used cached options if config object passed down is the same
    if (CACHED_OPTS.containsKey(conf)) {
      opts = CACHED_OPTS.get(conf);
    } else {
      opts = new org.rocksdb.Options();
      if (cacheSize > 0) {
        BlockBasedTableConfig tableConfig = new BlockBasedTableConfig();
        tableConfig.setBlockCacheSize(cacheSize);
        opts.setTableFormatConfig(tableConfig);
      }

      String rocksDbStat = conf.getTrimmed(
          OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
          OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT);

      if (!rocksDbStat.equals(OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF)) {
        Statistics statistics = new Statistics();
        statistics.setStatsLevel(StatsLevel.valueOf(rocksDbStat));
        opts = opts.setStatistics(statistics);
      }
    }
    opts.setCreateIfMissing(createIfMissing);
    CACHED_OPTS.put(conf, opts);
    return new RocksDBStore(dbFile, opts);
  }
  
  throw new IllegalArgumentException("Invalid argument for "
      + OzoneConfigKeys.OZONE_METADATA_STORE_IMPL
      + ". Expecting " + OZONE_METADATA_STORE_IMPL_LEVELDB
      + " or " + OZONE_METADATA_STORE_IMPL_ROCKSDB
      + ", but met " + dbType);
}
Example 18
Source File: LevelDbStore.java    From dubbox with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {
		Kryo kryo = new Kryo();
		kryo.setReferences(false);
		kryo.setRegistrationRequired(false);
		kryo.setInstantiatorStrategy(new DefaultInstantiatorStrategy(new StdInstantiatorStrategy()));
//		kryo.register(Transaction.class);
//		kryo.register(Xid.class);
//		kryo.register(TransactionType.class);
//		kryo.register(TransactionStatus.class);
//		kryo.register(Participant.class);
		
//		Output output = new Output(new ByteArrayOutputStream());//new Output(new FileOutputStream("file.bin"));
//		
//		DefaultTransaction defaultTransaction = new DefaultTransaction();
//		defaultTransaction.setXid(new DefaultXid("1", "2", "3"));
//		
//		kryo.writeObject(output, defaultTransaction);
//		output.flush();
//		output.close();
		
		Options options = new Options();
		options.createIfMissing(true);
		options.cacheSize(100 * 1048576); // 100MB cache
		options.logger(new org.iq80.leveldb.Logger() {
			public void log(String message) {
				TransactionLog.REPOSITORY.info(message);
			}
		});
		DB db = Iq80DBFactory.factory.open(new File("txtreedb"), options);
		
//		String stats = db.getProperty("leveldb.stats");
//		System.out.println(stats);
		
		DBIterator iterator = null;
		try {
			iterator = db.iterator();
			for(iterator.seekToFirst(); iterator.hasNext(); iterator.next()) {
				String key = Iq80DBFactory.asString(iterator.peekNext().getKey());
				Transaction value = (Transaction) SerializationUtil.deserialize(iterator.peekNext().getValue());
				
				System.out.println(key+" = "+value);
			}
		} finally {
			if (iterator != null) {
				iterator.close();
			}
		}

//		String transactionId = "f282205a-e794-4bda-83a2-bb28b8839cad";
//		Input input = new Input(db.get(Iq80DBFactory.bytes(transactionId)));
//		Transaction transaction = (Transaction) kryo.readClassAndObject(input);
//		System.out.println(transaction);
	}
Example 19
Source File: ReaderWithOffsets.java    From distributedlog with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {
    if (4 != args.length) {
        System.out.println(HELP);
        return;
    }

    String dlUriStr = args[0];
    final String streamName = args[1];
    final String readerId = args[2];
    final String offsetStoreFile = args[3];

    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder()
            .conf(conf)
            .uri(uri)
            .build();

    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);

    // open the offset store
    Options options = new Options();
    options.createIfMissing(true);
    final DB offsetDB = factory.open(new File(offsetStoreFile), options);
    final AtomicReference<DLSN> lastDLSN = new AtomicReference<DLSN>(null);
    // offset updater
    final ScheduledExecutorService executorService =
            Executors.newSingleThreadScheduledExecutor();
    executorService.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            if (null != lastDLSN.get()) {
                offsetDB.put(readerId.getBytes(UTF_8), lastDLSN.get().serializeBytes());
                System.out.println("Updated reader " + readerId + " offset to " + lastDLSN.get());
            }
        }
    }, 10, 10, TimeUnit.SECONDS);
    try {
        byte[] offset = offsetDB.get(readerId.getBytes(UTF_8));
        DLSN dlsn;
        if (null == offset) {
            dlsn = DLSN.InitialDLSN;
        } else {
            dlsn = DLSN.deserializeBytes(offset);
        }
        readLoop(dlm, dlsn, lastDLSN);
    } finally {
        offsetDB.close();
        dlm.close();
        namespace.close();
    }
}
Example 20
Source File: ReaderWithOffsets.java    From distributedlog with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {
    if (4 != args.length) {
        System.out.println(HELP);
        return;
    }

    String dlUriStr = args[0];
    final String streamName = args[1];
    final String readerId = args[2];
    final String offsetStoreFile = args[3];

    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf)
            .uri(uri)
            .build();

    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);

    // open the offset store
    Options options = new Options();
    options.createIfMissing(true);
    final DB offsetDB = factory.open(new File(offsetStoreFile), options);
    final AtomicReference<DLSN> lastDLSN = new AtomicReference<DLSN>(null);
    // offset updater
    final ScheduledExecutorService executorService =
            Executors.newSingleThreadScheduledExecutor();
    executorService.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            if (null != lastDLSN.get()) {
                offsetDB.put(readerId.getBytes(UTF_8), lastDLSN.get().serializeBytes());
                System.out.println("Updated reader " + readerId + " offset to " + lastDLSN.get());
            }
        }
    }, 10, 10, TimeUnit.SECONDS);
    try {
        byte[] offset = offsetDB.get(readerId.getBytes(UTF_8));
        DLSN dlsn;
        if (null == offset) {
            dlsn = DLSN.InitialDLSN;
        } else {
            dlsn = DLSN.deserializeBytes(offset);
        }
        readLoop(dlm, dlsn, lastDLSN);
    } finally {
        offsetDB.close();
        dlm.close();
        namespace.close();
    }
}