org.janusgraph.diskstorage.BackendException Java Examples

@Override
public Void call() throws BackendException {

    final UpdateItem updateBackoff = new UpdateItem(updateItemRequest, dynamoDbDelegate);
    final UpdateItemResult result = updateBackoff.runWithBackoff();


    final Map<String, AttributeValue> item = result.getAttributes();

    if (item == null) {
        // bail
        return null;
    }

    // If the record has no Titan columns left after deletions occur, then just delete the record
    if (item.containsKey(Constants.JANUSGRAPH_HASH_KEY) && item.size() == ATTRIBUTES_IN_EMPTY_SINGLE_ITEM) {
        final DeleteItem deleteBackoff = new DeleteItem(new DeleteItemRequest().withTableName(updateItemRequest.getTableName())
                                                                         .withKey(updateItemRequest.getKey()), dynamoDbDelegate);
        deleteBackoff.runWithBackoff();
    }

    // void
    return null;
}
 

public KeyColumnValueStoreManager openStorageManager() throws BackendException {
    final List<String> logNames = new ArrayList<>(12);
    logNames.add("test1");
    logNames.add("durable");
    logNames.add("ml0");
    logNames.add("ml1");
    logNames.add("ml2");
    logNames.add("loner0");
    logNames.add("loner1");
    logNames.add("loner2");
    logNames.add("loner3");
    logNames.add("loner4");
    logNames.add("fuzz");
    logNames.add("testx");
    final WriteConfiguration wc = TestGraphUtil.instance.getStoreConfig(model, logNames);
    final BasicConfiguration config = new BasicConfiguration(GraphDatabaseConfiguration.ROOT_NS, wc,
        BasicConfiguration.Restriction.NONE);

    return new DynamoDBStoreManager(config);
}
 

void createTableAndWaitForActive(final CreateTableRequest request) throws BackendException {
    final String tableName = request.getTableName();
    Preconditions.checkArgument(!Strings.isNullOrEmpty(tableName), "Table name was null or empty");
    final TableDescription desc;
    try {
        desc = this.describeTable(tableName);
        if (null != desc && isTableAcceptingWrites(desc.getTableStatus())) {
            return; //store existed
        }
    } catch (BackendNotFoundException e) {
        log.debug(tableName + " did not exist yet, creating it", e);
    }

    createTable(request);
    waitForTableCreation(tableName, false /*verifyIndexesList*/, null /*expectedLsiList*/, null /*expectedGsiList*/);
}
 

boolean ensureTableDeleted(final String tableName) throws BackendException {
    boolean successFlag = false;
    int retryCount = 0;
    do {
        try {
            this.describeTable(tableName);
        } catch (BackendNotFoundException e) {
            successFlag = true;
            break;
        }

        interruptibleSleep(CONTROL_PLANE_RETRY_DELAY_MS);
        retryCount++;
    } while (!successFlag && retryCount < maxRetries);
    if (!successFlag) {
        throw new PermanentBackendException("Table deletion not completed after retrying " + maxRetries + " times");
    }
    return successFlag;
}
 

@Override
public void mutate(final StaticBuffer hashKey, final List<Entry> additions, final List<StaticBuffer> deletions, final StoreTransaction txh) throws BackendException {
    log.debug("Entering mutate table:{} keys:{} additions:{} deletions:{} txh:{}",
              getTableName(),
              encodeKeyForLog(hashKey),
              encodeForLog(additions),
              encodeForLog(deletions),
              txh);
    super.mutateOneKey(hashKey, new KCVMutation(additions, deletions), txh);

    log.debug("Exiting mutate table:{} keys:{} additions:{} deletions:{} txh:{} returning:void",
              getTableName(),
              encodeKeyForLog(hashKey),
              encodeForLog(additions),
              encodeForLog(deletions),
              txh);
}
 

public A runWithBackoff() throws BackendException {
    boolean interrupted = false;
    try {
        do {
            interrupted = runWithBackoffOnce();
        } while (result == null);
        return result;
    } finally {
        //meter tries
        delegate.getMeter(delegate.getMeterName(apiNameRetries, getTableName())).mark(tries - 1);

        if (interrupted) {
            Thread.currentThread().interrupt();
            throw new BackendRuntimeException("exponential backoff was interrupted");
        }
    }
}
 

private boolean runWithBackoffOnce() throws BackendException {
    boolean interrupted = false;
    tries++;
    try {
        result = call();
    } catch (TemporaryBackendException e) { //retriable
        if (tries > delegate.getMaxRetries()) {
            throw new TemporaryBackendException("Max tries exceeded.", e);
        }
        try {
            Thread.sleep(exponentialBackoffTime);
        } catch (InterruptedException ie) {
            interrupted = true;
        } finally {
            exponentialBackoffTime *= 2;
        }
    }
    return interrupted;
}
 

UpdateItemResult updateItem(final UpdateItemRequest request) throws BackendException {
    setUserAgent(request);
    UpdateItemResult result;
    final int bytes;
    if (request.getUpdateExpression() != null) {
        bytes = calculateExpressionBasedUpdateSize(request);
    } else {
        bytes = calculateItemUpdateSizeInBytes(request.getAttributeUpdates());
    }
    getBytesHistogram(UPDATE_ITEM, request.getTableName()).update(bytes);
    final int wcu = computeWcu(bytes);
    timedWriteThrottle(UPDATE_ITEM, request.getTableName(), wcu);

    final Timer.Context apiTimerContext = getTimerContext(UPDATE_ITEM, request.getTableName());
    try {
        result = client.updateItem(request);
    } catch (Exception e) {
        throw processDynamoDbApiException(e, UPDATE_ITEM, request.getTableName());
    } finally {
        apiTimerContext.stop();
    }
    meterConsumedCapacity(UPDATE_ITEM, result.getConsumedCapacity());

    return result;
}
 

public ScanResult scan(final ScanRequest request, final int permitsToConsume) throws BackendException {
    setUserAgent(request);
    ScanResult result;
    timedReadThrottle(SCAN, request.getTableName(), permitsToConsume);

    final Timer.Context apiTimerContext = getTimerContext(SCAN, request.getTableName());
    try {
        result = client.scan(request);
    } catch (Exception e) {
        throw processDynamoDbApiException(e, SCAN, request.getTableName());
    } finally {
        apiTimerContext.stop();
    }
    meterConsumedCapacity(SCAN, result.getConsumedCapacity());
    measureItemCount(SCAN, request.getTableName(), result.getCount());
    return result;
}
 

@Override
public Deployment getDeployment() {
    if (null != deployment) {
        return deployment;
    }

    List<KeyRange> local;
    try {
        local = getLocalKeyPartition();
        deployment = null != local && !local.isEmpty() ? Deployment.LOCAL : Deployment.REMOTE;
    } catch (BackendException e) {
        throw new RuntimeException(e);
    }
    return deployment;
}
 

@Override
public void acquireLock(StaticBuffer key,
                        StaticBuffer column,
                        StaticBuffer expectedValue,
                        StoreTransaction txh) throws BackendException {
    throw new UnsupportedOperationException();
}
 

/**
 * This test is to demonstrate performance in response to a report of elevated latency for committing 30 vertices.
 * http://stackoverflow.com/questions/42899388/titan-dynamodb-local-incredibly-slow-8s-commit-for-30-vertices
 * @throws BackendException in case cleanUpTables fails
 */
@Test
public void performanceTest() throws BackendException {
    final Graph graph = JanusGraphFactory.open(TestGraphUtil.instance.createTestGraphConfig(BackendDataModel.MULTI));
    IntStream.of(30).forEach(i -> graph.addVertex(LABEL));
    final Stopwatch watch = Stopwatch.createStarted();
    graph.tx().commit();
    System.out.println("Committing took " + watch.stop().elapsed(TimeUnit.MILLISECONDS) + " ms");
    TestGraphUtil.instance.cleanUpTables();
}
 

/**
 * Deletes the specified table with all its columns.
 * ATTENTION: Invoking this method will delete the table if it exists and therefore causes data loss.
 */
@Override
public void clearStorage() throws BackendException {
    try (AdminMask adm = getAdminInterface()) {
        if (this.storageConfig.get(DROP_ON_CLEAR)) {
            adm.dropTable(tableName);
        } else {
            adm.clearTable(tableName, times.getTime(times.getTime()));
        }
    } catch (IOException e)
    {
        throw new TemporaryBackendException(e);
    }
}
 

@Override
public Map<StaticBuffer, EntryList> getSlice(final List<StaticBuffer> keys, final SliceQuery query, final StoreTransaction txh) throws BackendException {
    log.debug("Entering getSliceMultiSliceQuery table:{} keys:{} query:{} txh:{}",
              getTableName(),
              encodeForLog(keys),
              encodeForLog(query),
              txh);

    final Map<StaticBuffer, EntryList> resultMap = Maps.newHashMapWithExpectedSize(keys.size());

    final List<QueryWorker> queryWorkers = Lists.newLinkedList();
    for (StaticBuffer hashKey : keys) {
        final QueryWorker queryWorker = buildQueryWorker(hashKey, query);
        queryWorkers.add(queryWorker);

        resultMap.put(hashKey, EntryList.EMPTY_LIST);
    }

    final List<QueryResultWrapper> results = client.getDelegate().parallelQuery(queryWorkers);
    for (QueryResultWrapper resultWrapper : results) {
        final StaticBuffer titanKey = resultWrapper.getTitanKey();

        final QueryResult dynamoDBResult = resultWrapper.getDynamoDBResult();
        final EntryList entryList = createEntryListFromItems(dynamoDBResult.getItems(), query);
        resultMap.put(titanKey, entryList);
    }

    log.debug("Exiting getSliceMultiSliceQuery table:{} keys:{} query:{} txh:{} returning:{}",
              getTableName(),
              encodeForLog(keys),
              encodeForLog(query),
              txh,
              resultMap.size());
    return resultMap;
}
 

@Override
public EntryList getSlice(final KeySliceQuery query, final StoreTransaction txh)
        throws BackendException {
    log.debug("Entering getSliceKeySliceQuery table:{} query:{} txh:{}", getTableName(), encodeForLog(query), txh);
    final EntryList result = getKeysRangeQuery(query.getKey(), query, txh);
    log.debug("Exiting getSliceKeySliceQuery table:{} query:{} txh:{} returning:{}", getTableName(), encodeForLog(query), txh,
              result.size());
    return result;
}
 

@Override
public void close() throws BackendException {
    logger.trace("Shutting down connection to Solr", solrClient);
    try {
        solrClient.close();
    } catch (final IOException e) {
        throw new TemporaryBackendException(e);
    }
}
 

@Override
public void rollback() throws BackendException {
    log.debug("rollback id:{}", id);
    releaseLocks();
    expectedValues.clear();
    super.rollback();
}
 

/**
 * The main code basically instantiate its own class and call individual methods.
 * @param argv
 */
public static void main(String[] argv) throws BackendException, ExecutionException, InterruptedException {
  // conect the graph
  RecreateWeightIndex schema = new RecreateWeightIndex(Schema.CONFIG_FILE);

  schema.deleteOldIndexes();
  schema.createNewIndexes();
  schema.commit();
  schema.waitForIndexes();
  //schema.reindex();

  schema.close();
}
 

@Override
public EntryList getSlice(final KeySliceQuery query, final StoreTransaction txh) throws BackendException {
    log.debug("Entering getSliceKeySliceQuery table:{} query:{} txh:{}", getTableName(), encodeForLog(query), txh);
    final GetItemRequest request = super.createGetItemRequest().withKey(new ItemBuilder().hashKey(query.getKey()).build());
    final GetItemResult result = new ExponentialBackoff.GetItem(request, client.getDelegate()).runWithBackoff();

    final List<Entry> filteredEntries = extractEntriesFromGetItemResult(result, query.getSliceStart(), query.getSliceEnd(), query.getLimit());
    log.debug("Exiting getSliceKeySliceQuery table:{} query:{} txh:{} returning:{}", getTableName(), encodeForLog(query), txh,
              filteredEntries.size());
    return StaticArrayEntryList.of(filteredEntries);
}
 

@Override
public void commit() throws BackendException {
    log.debug("commit id:{}", id);
    releaseLocks();
    expectedValues.clear();
    super.commit();
}
 

private void reindexFor(String label, String propertyKey) throws BackendException, ExecutionException, InterruptedException {
  LOGGER.info("Reindexing index for edge {} and property {} using map reduce", label, propertyKey);

  MapReduceIndexManagement mr = new MapReduceIndexManagement(graph);
  JanusGraphIndex index = mgt.getGraphIndex(Schema.indexName(label, propertyKey));
  mr.updateIndex(index, SchemaAction.REINDEX).get();
}
 

/**
 * Paginates through pages of an entity.
 * @return an encapsulation of the pages of an entity
 * @throws BackendException if there was any trouble paginating
 */
public R call() throws BackendException {
    while (hasNext) {
        pagesProcessed++;
        next();
    }
    delegate.updatePagesHistogram(apiName, tableName, pagesProcessed);
    return getMergedPages();
}
 

@Override
public ListTablesResult next() throws BackendException {
    final ListTablesResult result = delegate.listTables(request);
    if (result.getLastEvaluatedTableName() != null && !result.getLastEvaluatedTableName().isEmpty()) {
        request.setExclusiveStartTableName(result.getLastEvaluatedTableName());
    } else { //done
        markComplete();
    }

    // c add scanned items
    tableNames.addAll(result.getTableNames());
    return result;
}
 

private CreateTableResult createTable(final CreateTableRequest request) throws BackendException {
    controlPlaneRateLimiter.acquire();
    final Timer.Context apiTimerContext = getTimerContext(CREATE_TABLE, request.getTableName());
    CreateTableResult result;
    try {
        result = client.createTable(request);
    } catch (final Exception e) {
        throw processDynamoDbApiException(e, CREATE_TABLE, request.getTableName());
    } finally {
        apiTimerContext.stop();
    }
    return result;
}
 

@Override
public final void deleteStore() throws BackendException {
    log.debug("Entering deleteStore name:{}", name);
    client.getDelegate().deleteTable(getTableSchema().getTableName());
    //block until the tables are actually deleted
    client.getDelegate().ensureTableDeleted(getTableSchema().getTableName());
}
 

public KeyColumnValueStoreManager openStorageManager() throws BackendException {
    final List<String> storeNames = new ArrayList<>(2);
    storeNames.add(storeName1);
    storeNames.add(storeName2);
    final WriteConfiguration wc = TestGraphUtil.instance.getStoreConfig(model, storeNames);
    final BasicConfiguration config = new BasicConfiguration(GraphDatabaseConfiguration.ROOT_NS, wc,
        BasicConfiguration.Restriction.NONE);

    return new DynamoDBStoreManager(config);
}
 

@Override
public KeyColumnValueStoreManager openStorageManager() throws BackendException
{
    final List<String> storeNames = Collections.singletonList("testStore1");
    final WriteConfiguration wc = TestGraphUtil.instance.getStoreConfig(model, storeNames);

    if (name.getMethodName().equals("parallelScanTest")) {
        wc.set("storage.dynamodb." + Constants.DYNAMODB_ENABLE_PARALLEL_SCAN.getName(), "true");
    }
    final BasicConfiguration config = new BasicConfiguration(GraphDatabaseConfiguration.ROOT_NS, wc,
        BasicConfiguration.Restriction.NONE);

    return new DynamoDBStoreManager(config);
}
 

ListTablesResult listTables(final ListTablesRequest request) throws BackendException {
    controlPlaneRateLimiter.acquire();
    final Timer.Context apiTimerContext = getTimerContext(listTablesApiName, null /*tableName*/);
    ListTablesResult result;
    try {
        result = client.listTables(request);
    } catch (final Exception e) {
        throw processDynamoDbApiException(e, LIST_TABLES, null /*tableName*/);
    } finally {
        apiTimerContext.stop();
    }
    return result;
}
 

@Test
public void processTripleWithMaps() throws BackendException {
    final ConcurrentMap<String, Vertex> brandTypeMap = new ConcurrentHashMap<>();
    final ConcurrentMap<String, Vertex> companyMap = new ConcurrentHashMap<>();
    final ConcurrentMap<String, Vertex> hotelBrandMap = new ConcurrentHashMap<>();
    tripleIngestBase((StandardJanusGraph graph, List<Triple> triples) -> {
        final JanusGraphTransaction threadedGraph = graph.newTransaction();
        triples.parallelStream().forEach(triple -> {
            final Vertex outV = hotelBrandMap.computeIfAbsent(triple.getLeftPropertyValue(),
                value -> threadedGraph.addVertex(triple.getLeftPropertyName(), value));
            //your original method was creating vertices in the processRelationship method.
            //this caused the uniqueness constraint violation (one of a few issues in your
            //original code) because you have a unique index on the rightPropertyName
            switch (triple.getRelationship()) {
                case hotelBrandType:
                    outV.addEdge("hotelBrandType",brandTypeMap.computeIfAbsent(triple.getRightPropertyValue(),
                        value -> threadedGraph.addVertex(triple.getRightPropertyName(), value)));
                    break;
                case instanceOf:
                    outV.addEdge("instanceOf", companyMap.computeIfAbsent(triple.getRightPropertyValue(),
                        value -> threadedGraph.addVertex(triple.getRightPropertyName(), value)));
                    break;
                default:
                    throw new IllegalArgumentException("unexpected relationship type");
            }
        });
        final Stopwatch watch = Stopwatch.createStarted();
        threadedGraph.commit();
        log.info("Committed in " + watch.elapsed(TimeUnit.MILLISECONDS) + " ms");
        watch.stop();
    });
}
 

ParallelScanner getParallelScanCompletionService(final ScanRequest initialRequest) throws BackendException {
    final int segments = Math.max(1, clientThreadPool.getMaximumPoolSize() / maxConcurrentUsers);
    final ParallelScanner completion = new ParallelScanner(clientThreadPool, segments, this);

    for (int segment = 0; segment < segments; segment++) {
        // dont need to set user agent here because ExponentialBackoff.Scan
        // calls DynamoDbDelegate.scan which sets it
        final ScanRequest scanSegment = copyScanRequest(initialRequest).withTotalSegments(segments).withSegment(segment);
        completion.addWorker(new ScanSegmentWorker(this, scanSegment), segment);
    }

    return completion;
}