Java Code Examples for org.rocksdb.RocksDBException#getMessage()

@Override
public V get(TimeWindow window, K key) {
    try {
        ColumnFamilyHandle handler = getColumnFamilyHandle(window);
        V ret = null;
        if (key != null) {
            byte[] rawKey = serializer.serialize(key);
            byte[] rawData = rocksDb.get(handler, rawKey);
            ret = rawData != null ? (V) serializer.deserialize(rawData) : null;
        }
        return ret;
    } catch (RocksDBException e) {
        LOG.error("Failed to get value by key-{} for timeWindow={}", key, window);
        throw new RuntimeException(e.getMessage());
    }
}
 

@Override
public Collection<K> getAllKeys(TimeWindow window) {
    try {
        ColumnFamilyHandle handler = getColumnFamilyHandle(window);
        Collection<K> keys = new ArrayList<K>();
        RocksIterator itr = rocksDb.newIterator(handler);
        itr.seekToFirst();
        while (itr.isValid()) {
            keys.add((K) serializer.deserialize(itr.key()));
            itr.next();
        }
        return keys;
    } catch (RocksDBException e) {
        LOG.error("Failed to get all keys for timeWindow={}", window);
        throw new RuntimeException(e.getMessage());
    }
}
 

/**
 * Flush the data in memtable of RocksDB into disk, and then create checkpoint
 * 
 * @param batchId
 */
@Override
public void checkpoint(long batchId) {
    long startTime = System.currentTimeMillis();
    try {
        rocksDb.flush(new FlushOptions());
        Checkpoint cp = Checkpoint.create(rocksDb);
        cp.createCheckpoint(getLocalCheckpointPath(batchId));
    } catch (RocksDBException e) {
        LOG.error("Failed to create checkpoint for batch-" + batchId, e);
        throw new RuntimeException(e.getMessage());
    }

    if (JStormMetrics.enabled)
        rocksDbFlushAndCpLatency.update(System.currentTimeMillis() - startTime);
}
 

public RocksDBStore(File dbFile, Options options) throws IOException {
  Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
  RocksDB.loadLibrary();
  dbOptions = options;
  dbLocation = dbFile;
  writeOptions = new WriteOptions();
  try {
    db = RocksDB.open(dbOptions, dbLocation.getAbsolutePath());
    if (dbOptions.statistics() != null) {
      Map<String, String> jmxProperties = new HashMap<String, String>();
      jmxProperties.put("dbName", dbFile.getName());
      statMBeanName = HddsUtils.registerWithJmxProperties(
          "Ozone", "RocksDbStore", jmxProperties,
          RocksDBStoreMBean.create(dbOptions.statistics(), dbFile.getName()));
      if (statMBeanName == null) {
        LOG.warn("jmx registration failed during RocksDB init, db path :{}",
            dbFile.getAbsolutePath());
      }
    }
  } catch (RocksDBException e) {
    String msg = "Failed init RocksDB, db path : " + dbFile.getAbsolutePath()
        + ", " + "exception :" + (e.getCause() == null ?
        e.getClass().getCanonicalName() + " " + e.getMessage() :
        e.getCause().getClass().getCanonicalName() + " " +
            e.getCause().getMessage());
    throw new IOException(msg, e);
  }

  if (LOG.isDebugEnabled()) {
    LOG.debug("RocksDB successfully opened.");
    LOG.debug("[Option] dbLocation= {}", dbLocation.getAbsolutePath());
    LOG.debug("[Option] createIfMissing = {}", options.createIfMissing());
    LOG.debug("[Option] compactionPriority= {}", options.compactionStyle());
    LOG.debug("[Option] compressionType= {}", options.compressionType());
    LOG.debug("[Option] maxOpenFiles= {}", options.maxOpenFiles());
    LOG.debug("[Option] writeBufferSize= {}", options.writeBufferSize());
  }
}
 

public static IOException toIOException(String msg, RocksDBException e) {
  String statusCode = e.getStatus() == null ? "N/A" :
      e.getStatus().getCodeString();
  String errMessage = e.getMessage() == null ? "Unknown error" :
      e.getMessage();
  String output = msg + "; status : " + statusCode
      + "; message : " + errMessage;
  return new IOException(output, e);
}
 

public static IOException toIOException(String msg, RocksDBException e) {
  String statusCode = e.getStatus() == null ? "N/A" :
      e.getStatus().getCodeString();
  String errMessage = e.getMessage() == null ? "Unknown error" :
      e.getMessage();
  String output = msg + "; status : " + statusCode
      + "; message : " + errMessage;
  return new IOException(output, e);
}
 

/**
 * Converts RocksDB exception to IOE.
 * @param msg  - Message to add to exception.
 * @param e - Original Exception.
 * @return  IOE.
 */
public static IOException toIOException(String msg, RocksDBException e) {
  String statusCode = e.getStatus() == null ? "N/A" :
      e.getStatus().getCodeString();
  String errMessage = e.getMessage() == null ? "Unknown error" :
      e.getMessage();
  String output = msg + "; status : " + statusCode
      + "; message : " + errMessage;
  return new IOException(output, e);
}
 

@Override
public void put(TimeWindow window, Object key, Object value) {
    try {
        ColumnFamilyHandle handler = getColumnFamilyHandle(window);
        rocksDb.put(handler, serializer.serialize(key), serializer.serialize(value));
    } catch (RocksDBException e) {
        LOG.error("Failed to put data, key={}, value={}, for timeWindow={}", key, value, window);
        throw new RuntimeException(e.getMessage());
    }
}
 

@Override
public void putBatch(TimeWindow window, Map<K, V> batch) {
    try {
        ColumnFamilyHandle handler = getColumnFamilyHandle(window);
        WriteBatch writeBatch = new WriteBatch();
        for (Map.Entry<K, V> entry : batch.entrySet()) {
            writeBatch.put(handler, serializer.serialize(entry.getKey()), serializer.serialize(entry.getValue()));
        }
        rocksDb.write(new WriteOptions(), writeBatch);
    } catch (RocksDBException e) {
        LOG.error("Failed to put batch={} for window={}", batch, window);
        throw new RuntimeException(e.getMessage());
    }
}
 

protected void initRocksDb() {
    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }

    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        if (ttlTimeSec > 0)
            rocksDb = TtlDB.open(options, rocksDbDir, ttlTimeSec, false);
        else
            rocksDb = RocksDB.open(options, rocksDbDir);
        // enable compaction
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }

    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}
 

@Override
public void put(K key, V value) {
    try {
        rocksDb.put(serializer.serialize(key), serializer.serialize(value));
    } catch (RocksDBException e) {
        LOG.error("Failed to put data, key={}, value={}", key, value);
        throw new RuntimeException(e.getMessage());
    }
}
 

@Override
public void putBatch(Map<K, V> batch) {
    try {
        WriteBatch writeBatch = new WriteBatch();
        for (Map.Entry<K, V> entry : batch.entrySet()) {
            writeBatch.put(serializer.serialize(entry.getKey()), serializer.serialize(entry.getValue()));
        }
        rocksDb.write(new WriteOptions(), writeBatch);
    } catch (RocksDBException e) {
        LOG.error("Failed to put batch={}", batch);
        throw new RuntimeException(e.getMessage());
    }
}
 

@Override
public V get(K key) {
    try {
        V ret = null;
        if (key != null) {
            byte[] rawKey = serializer.serialize(key);
            byte[] rawData = rocksDb.get(rawKey);
            ret = rawData != null ? (V) serializer.deserialize(rawData) : null;
        }
        return ret;
    } catch (RocksDBException e) {
        LOG.error("Failed to get value by key-{}", key);
        throw new RuntimeException(e.getMessage());
    }
}
 

public RDBStore(File dbFile, DBOptions options,
    WriteOptions writeOptions, Set<TableConfig> families,
                CodecRegistry registry)
    throws IOException {
  Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
  Preconditions.checkNotNull(families);
  Preconditions.checkArgument(!families.isEmpty());
  handleTable = new HashMap<>();
  codecRegistry = registry;
  final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
      new ArrayList<>();
  columnFamilyHandles = new ArrayList<>();

  for (TableConfig family : families) {
    columnFamilyDescriptors.add(family.getDescriptor());
  }

  dbOptions = options;
  dbLocation = dbFile;
  this.writeOptions = writeOptions;

  try {
    // This logic has been added to support old column families that have
    // been removed, or those that may have been created in a future version.
    // TODO : Revisit this logic during upgrade implementation.
    List<TableConfig> columnFamiliesInDb = getColumnFamiliesInExistingDb();
    List<TableConfig> extraCf = columnFamiliesInDb.stream().filter(
        cf -> !families.contains(cf)).collect(Collectors.toList());
    if (!extraCf.isEmpty()) {
      LOG.info("Found the following extra column families in existing DB : " +
              "{}", extraCf);
      extraCf.forEach(cf -> columnFamilyDescriptors.add(cf.getDescriptor()));
    }

    db = RocksDB.open(dbOptions, dbLocation.getAbsolutePath(),
        columnFamilyDescriptors, columnFamilyHandles);

    for (int x = 0; x < columnFamilyHandles.size(); x++) {
      handleTable.put(
          StringUtils.bytes2String(columnFamilyHandles.get(x).getName()),
          columnFamilyHandles.get(x));
    }

    if (dbOptions.statistics() != null) {
      Map<String, String> jmxProperties = new HashMap<>();
      jmxProperties.put("dbName", dbFile.getName());
      statMBeanName = HddsUtils.registerWithJmxProperties(
          "Ozone", "RocksDbStore", jmxProperties,
          RocksDBStoreMBean.create(dbOptions.statistics(),
              dbFile.getName()));
      if (statMBeanName == null) {
        LOG.warn("jmx registration failed during RocksDB init, db path :{}",
            dbFile.getAbsolutePath());
      }
    }

    //create checkpoints directory if not exists.
    checkpointsParentDir =
            Paths.get(dbLocation.getParent(), "db.checkpoints").toString();
    File checkpointsDir = new File(checkpointsParentDir);
    if (!checkpointsDir.exists()) {
      boolean success = checkpointsDir.mkdir();
      if (!success) {
        LOG.warn("Unable to create RocksDB checkpoint directory");
      }
    }

    //Initialize checkpoint manager
    checkPointManager = new RDBCheckpointManager(db, "rdb");
    rdbMetrics = RDBMetrics.create();

  } catch (RocksDBException e) {
    String msg = "Failed init RocksDB, db path : " + dbFile.getAbsolutePath()
        + ", " + "exception :" + (e.getCause() == null ?
        e.getClass().getCanonicalName() + " " + e.getMessage() :
        e.getCause().getClass().getCanonicalName() + " " +
            e.getCause().getMessage());

    throw toIOException(msg, e);
  }

  if (LOG.isDebugEnabled()) {
    LOG.debug("RocksDB successfully opened.");
    LOG.debug("[Option] dbLocation= {}", dbLocation.getAbsolutePath());
    LOG.debug("[Option] createIfMissing = {}", options.createIfMissing());
    LOG.debug("[Option] maxOpenFiles= {}", options.maxOpenFiles());
  }
}
 

@Override
protected void initRocksDb() {
    windowToCFHandler = new HashMap<>();

    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }

    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        List<Integer> ttlValues = new ArrayList<>();

        List<byte[]> families = RocksDB.listColumnFamilies(options, rocksDbDir);
        List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, cfOptions));
                LOG.debug("Load colum family of {}", new String(bytes));
                if (ttlTimeSec > 0)
                    ttlValues.add(ttlTimeSec);
            }
        }
        
        if (columnFamilyDescriptors.size() > 0) {
            if (ttlTimeSec > 0)
                rocksDb = TtlDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles, ttlValues, false);
            else
                rocksDb = RocksDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles);

            int n = Math.min(columnFamilyDescriptors.size(), columnFamilyHandles.size());
            LOG.info("Try to load RocksDB with column family, desc_num={}, handler_num={}", columnFamilyDescriptors.size(), columnFamilyHandles.size());
            // skip default column
            for (int i = 1; i < n; i++) {
                windowToCFHandler.put((TimeWindow) serializer.deserialize(columnFamilyDescriptors.get(i).columnFamilyName()), columnFamilyHandles.get(i));
            }
        } else {
            rocksDb = RocksDB.open(options, rocksDbDir);
        }
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }

    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}