org.apache.accumulo.core.client.Connector Java Examples

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

public static void setQueryInfo(Job job, Set<String> entityTypes, Node query, EntityShardBuilder shardBuilder, TypeRegistry<String> typeRegistry) throws AccumuloSecurityException, AccumuloException, TableNotFoundException, IOException {

        validateOptions(job);

        checkNotNull(shardBuilder);
        checkNotNull(query);
        checkNotNull(entityTypes);
        checkNotNull(typeRegistry);

        Instance instance = getInstance(job);
        Connector connector = instance.getConnector(getPrincipal(job), getAuthenticationToken(job));
        BatchScanner scanner = connector.createBatchScanner(DEFAULT_IDX_TABLE_NAME, getScanAuthorizations(job), 5);
        GlobalIndexVisitor globalIndexVisitor = new EntityGlobalIndexVisitor(scanner, shardBuilder, entityTypes);

        configureScanner(job, entityTypes, query, new NodeToJexl(typeRegistry), globalIndexVisitor, typeRegistry, OptimizedQueryIterator.class);

        job.getConfiguration().setBoolean(QUERY, true);
        job.getConfiguration().set(TYPE_REGISTRY, new String(toBase64(typeRegistry)));
    }
 

public static void printTable(final Connector conn, final Authorizations authorizations, final String tableName, final PrintStream out)
                throws TableNotFoundException {
    final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd HHmmss");
    
    final StringBuilder sb = new StringBuilder("--Begin entire " + tableName + " table--");
    
    sb.append("\n");
    
    final Scanner scanner = conn.createScanner(tableName, authorizations);
    for (final Entry<Key,Value> e : scanner) {
        sb.append(e.getKey().toStringNoTime());
        sb.append(' ');
        sb.append(dateFormat.format(new Date(e.getKey().getTimestamp())));
        sb.append('\t');
        sb.append(getPrintableValue(e.getValue()));
        sb.append("\n");
    }
    
    sb.append("--End entire ").append(tableName).append(" table--").append("\n");
    
    out.println(sb);
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    tops.create(TableName.DATE_INDEX);
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

public void dropGraph() throws Exception {
    Connector connector = createConnector();
    AccumuloGraphTestUtils.ensureTableExists(connector, GraphConfiguration.DEFAULT_TABLE_NAME_PREFIX);
    AccumuloGraphTestUtils.dropGraph(connector, AccumuloGraph.getDataTableName(GraphConfiguration.DEFAULT_TABLE_NAME_PREFIX));
    AccumuloGraphTestUtils.dropGraph(connector, AccumuloGraph.getVerticesTableName(GraphConfiguration.DEFAULT_TABLE_NAME_PREFIX));
    AccumuloGraphTestUtils.dropGraph(connector, AccumuloGraph.getHistoryVerticesTableName(GraphConfiguration.DEFAULT_TABLE_NAME_PREFIX));
    AccumuloGraphTestUtils.dropGraph(connector, AccumuloGraph.getEdgesTableName(GraphConfiguration.DEFAULT_TABLE_NAME_PREFIX));
    AccumuloGraphTestUtils.dropGraph(connector, AccumuloGraph.getExtendedDataTableName(GraphConfiguration.DEFAULT_TABLE_NAME_PREFIX));
    AccumuloGraphTestUtils.dropGraph(connector, AccumuloGraph.getHistoryEdgesTableName(GraphConfiguration.DEFAULT_TABLE_NAME_PREFIX));
    AccumuloGraphTestUtils.dropGraph(connector, AccumuloGraph.getMetadataTableName(GraphConfiguration.DEFAULT_TABLE_NAME_PREFIX));
    connector.securityOperations().changeUserAuthorizations(
        AccumuloGraphConfiguration.DEFAULT_ACCUMULO_USERNAME,
        new org.apache.accumulo.core.security.Authorizations(
            VISIBILITY_A_STRING,
            VISIBILITY_B_STRING,
            VISIBILITY_C_STRING,
            VISIBILITY_MIXED_CASE_STRING
        )
    );
}
 

/**
 * Uses the specified tableName, store config, and maxVersions
 *
 * @param connector
 */
public AccumuloLastNStore(Connector connector, String tableName, StoreConfig config, int maxVersions, TypeRegistry<String> typeRegistry) throws TableNotFoundException, TableExistsException, AccumuloSecurityException, AccumuloException {
    checkNotNull(connector);
    checkNotNull(tableName);
    checkNotNull(config);
    checkNotNull(typeRegistry);

    this.connector = connector;
    this.tableName = tableName;
    this.typeRegistry = typeRegistry;

    if (!connector.tableOperations().exists(this.tableName)) {
        connector.tableOperations().create(this.tableName, true);
        configureTable(connector, this.tableName, maxVersions);
    }

    this.writer = this.connector.createBatchWriter(this.tableName, config.getMaxMemory(), config.getMaxLatency(), config.getMaxWriteThreads());
}
 

public MockRdfCloudStore() {
    super();
    Instance instance = new MockInstance();
    try {
        Connector connector = instance.getConnector("", "");
        setConf(new AccumuloRdfConfiguration());
        AccumuloRyaDAO cdao = new AccumuloRyaDAO();
        cdao.setConnector(connector);
        setRyaDAO(cdao);
        inferenceEngine = new InferenceEngine();
        inferenceEngine.setRyaDAO(cdao);
        inferenceEngine.setRefreshGraphSchedule(1000); //every sec
        setInferenceEngine(inferenceEngine);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

/**
 * Gets the AccumuloConnector singleton, starting the MiniAccumuloCluster on initialization.
 * This singleton instance is required so all test cases access the same MiniAccumuloCluster.
 *
 * @return Accumulo connector
 */
public static Connector getAccumuloConnector()
{
    if (connector != null) {
        return connector;
    }

    try {
        MiniAccumuloCluster accumulo = createMiniAccumuloCluster();
        Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
        connector = instance.getConnector(MAC_USER, new PasswordToken(MAC_PASSWORD));
        LOG.info("Connection to MAC instance %s at %s established, user %s password %s", accumulo.getInstanceName(), accumulo.getZooKeepers(), MAC_USER, MAC_PASSWORD);
        return connector;
    }
    catch (AccumuloException | AccumuloSecurityException | InterruptedException | IOException e) {
        throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get connector to Accumulo", e);
    }
}
 

@Test
public void dropPCJ() throws AccumuloException, AccumuloSecurityException, PCJStorageException, NotInitializedException, RyaDetailsRepositoryException {
    // Setup the PCJ storage that will be tested against.
    final Connector connector = getClusterInstance().getConnector();
    final String ryaInstanceName = testInstance.getRyaInstanceName();
    try(final PrecomputedJoinStorage pcjStorage =  new AccumuloPcjStorage(connector, ryaInstanceName)) {
        // Create a PCJ.
        final String pcjId = pcjStorage.createPcj("SELECT * WHERE { ?a <http://isA> ?b } ");

        // Delete the PCJ that was just created.
        pcjStorage.dropPcj(pcjId);

        // Ensure the Rya details have been updated to no longer include the PCJ's ID.
        final RyaDetailsRepository detailsRepo = new AccumuloRyaInstanceDetailsRepository(connector, ryaInstanceName);

        final ImmutableMap<String, PCJDetails> detailsMap = detailsRepo.getRyaInstanceDetails()
                .getPCJIndexDetails()
                .getPCJDetails();

        assertFalse( detailsMap.containsKey(pcjId) );
    }
}
 

@Override
public Scanner getScanner(Connector conn)
{
  Authorizations auths = new Authorizations();
  Scanner scan = null;
  try {
    scan = conn.createScanner(getStore().getTableName(), auths);
  } catch (TableNotFoundException e) {
    logger.error("table not found ");
    DTThrowable.rethrow(e);
  }
  scan.setRange(new Range());
  // scan.fetchColumnFamily("attributes");

  return scan;
}
 

@Inject
public ColumnCardinalityCache(Connector connector, AccumuloConfig config)
{
    this.connector = requireNonNull(connector, "connector is null");
    int size = requireNonNull(config, "config is null").getCardinalityCacheSize();
    Duration expireDuration = config.getCardinalityCacheExpiration();

    // Create a bounded executor with a pool size at 4x number of processors
    this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("cardinality-lookup-%s"));
    this.executorService = new BoundedExecutor(coreExecutor, 4 * Runtime.getRuntime().availableProcessors());

    LOG.debug("Created new cache size %d expiry %s", size, expireDuration);
    cache = CacheBuilder.newBuilder()
            .maximumSize(size)
            .expireAfterWrite(expireDuration.toMillis(), MILLISECONDS)
            .build(new CardinalityCacheLoader());
}
 

static void setUpRya() throws AccumuloException, AccumuloSecurityException, RyaDAOException {
    MockInstance mock = new MockInstance(INSTANCE_NAME);
    Connector conn = mock.getConnector(USERNAME, new PasswordToken(USERP));
    AccumuloRyaDAO dao = new AccumuloRyaDAO();
    dao.setConnector(conn);
    AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration();
    conf.setTablePrefix(PREFIX);
    dao.setConf(conf);
    dao.init();
    String ns = "http://example.com/";
    dao.add(new RyaStatement(new RyaIRI(ns+"s1"), new RyaIRI(ns+"p1"), new RyaIRI(ns+"o1")));
    dao.add(new RyaStatement(new RyaIRI(ns+"s1"), new RyaIRI(ns+"p2"), new RyaIRI(ns+"o2")));
    dao.add(new RyaStatement(new RyaIRI(ns+"s2"), new RyaIRI(ns+"p1"), new RyaIRI(ns+"o3"),
            new RyaIRI(ns+"g1")));
    dao.add(new RyaStatement(new RyaIRI(ns+"s3"), new RyaIRI(ns+"p3"), new RyaIRI(ns+"o3"),
            new RyaIRI(ns+"g2")));
    dao.destroy();
}
 

/**
 * Get an {@link Iterator} over the {@link BindingSet}s that are stored in the PCJ table.
 *
 * @param accumuloConn - A connection to the Accumulo that hsots the PCJ table. (not null)
 * @param pcjTableName - The name of the PCJ table that will be scanned. (not null)
 * @param auths - the user's authorizations that will be used to scan the table. (not null)
 * @return An iterator over all of the {@link BindingSet}s that are stored as
 *   results for the PCJ.
 * @throws PCJStorageException The binding sets could not be fetched.
 */
public CloseableIterator<BindingSet> listResults(final Connector accumuloConn, final String pcjTableName, final Authorizations auths) throws PCJStorageException {
    requireNonNull(pcjTableName);

    // Fetch the Variable Orders for the binding sets and choose one of them. It
    // doesn't matter which one we choose because they all result in the same output.
    final PcjMetadata metadata = getPcjMetadata(accumuloConn, pcjTableName);
    final VariableOrder varOrder = metadata.getVarOrders().iterator().next();

    try {
        // Fetch only the Binding Sets whose Variable Order matches the selected one.
        final Scanner scanner = accumuloConn.createScanner(pcjTableName, auths);
        scanner.fetchColumnFamily( new Text(varOrder.toString()) );

        // Return an Iterator that uses that scanner.
        return new ScannerBindingSetIterator(scanner, varOrder);

    } catch (final TableNotFoundException e) {
        throw new PCJStorageException(String.format("PCJ Table does not exist for name '%s'.", pcjTableName), e);
    }
}
 

/**
 * Constructs an instance of {@link ProspectorService}.
 *
 * @param connector - The Accumulo connector used to communicate with the table. (not null)
 * @param tableName - The name of the Accumulo table that will be queried for Prospect results. (not null)
 * @throws AccumuloException A problem occurred while creating the table.
 * @throws AccumuloSecurityException A problem occurred while creating the table.
 */
public ProspectorService(Connector connector, String tableName) throws AccumuloException, AccumuloSecurityException {
    this.connector = requireNonNull(connector);
    this.tableName = requireNonNull(tableName);

    this.plans = ProspectorUtils.planMap(manager.getPlans());

    // Create the table if it doesn't already exist.
    try {
        final TableOperations tos = connector.tableOperations();
        if(!tos.exists(tableName)) {
            tos.create(tableName);
        }
    } catch(TableExistsException e) {
        // Do nothing. Something else must have made it while we were.
    }
}
 

public static void deleteIndexTables(final Connector accCon, final int tableNum,
        final String prefix) throws AccumuloException, AccumuloSecurityException,
TableNotFoundException {
    final TableOperations ops = accCon.tableOperations();
    final String tablename = prefix + "INDEX_";
    for (int i = 1; i < tableNum + 1; i++) {
        if (ops.exists(tablename + i)) {
            ops.delete(tablename + i);
        }
    }
}
 

private void tableCheck(Connector c) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException {
    if (!c.tableOperations().exists(TABLE_NAME)) {
        c.tableOperations().create(TABLE_NAME);
        // Remove the versioning iterator
        c.tableOperations().removeProperty(TABLE_NAME, "table.iterator.majc.vers");
        c.tableOperations().removeProperty(TABLE_NAME, "table.iterator.majc.vers.opt.maxVersions");
        c.tableOperations().removeProperty(TABLE_NAME, "table.iterator.minc.vers");
        c.tableOperations().removeProperty(TABLE_NAME, "table.iterator.minc.vers.opt.maxVersions");
        c.tableOperations().removeProperty(TABLE_NAME, "table.iterator.scan.vers");
        c.tableOperations().removeProperty(TABLE_NAME, "table.iterator.scan.vers.opt.maxVersions");
    }
    if (!c.tableOperations().exists(INDEX_TABLE_NAME))
        c.tableOperations().create(INDEX_TABLE_NAME);
}
 

public QueryTestTableHelper(Connector connector, Logger log) throws AccumuloSecurityException, AccumuloException, TableExistsException,
                TableNotFoundException {
    // create mock instance and connector
    this.connector = connector;
    this.log = log;
    createTables();
}
 

public AccumuloEventStore(Connector connector, String indexTable, String shardTable, StoreConfig config, TypeRegistry<String> typeRegistry,
    EventShardBuilder shardBuilder)
    throws TableExistsException, AccumuloSecurityException, AccumuloException, TableNotFoundException {
    checkNotNull(connector);
    checkNotNull(indexTable);
    checkNotNull(shardTable);
    checkNotNull(config);
    checkNotNull(typeRegistry);

    this.shardBuilder = shardBuilder;

    KeyValueIndex<Event> keyValueIndex = new KeyValueIndex(connector, indexTable, shardBuilder, config, typeRegistry);

    helper = new EventQfdHelper(connector, indexTable, shardTable, config, shardBuilder, typeRegistry, keyValueIndex);
}
 

public static BatchWriter createDefaultBatchWriter(final String tablename, final Configuration conf)
        throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
    final Long DEFAULT_MAX_MEMORY = getWriterMaxMemory(conf);
    final Long DEFAULT_MAX_LATENCY = getWriterMaxLatency(conf);
    final Integer DEFAULT_MAX_WRITE_THREADS = getWriterMaxWriteThreads(conf);
    final Connector connector = ConfigUtils.getConnector(conf);
    return connector.createBatchWriter(tablename, DEFAULT_MAX_MEMORY, DEFAULT_MAX_LATENCY, DEFAULT_MAX_WRITE_THREADS);
}
 

@Override
protected BaseQueryResponse runTestQueryWithGrouping(Map<String,Integer> expected, String querystr, Date startDate, Date endDate,
                Map<String,String> extraParms, RebuildingScannerTestHelper.TEARDOWN teardown, RebuildingScannerTestHelper.INTERRUPT interrupt)
                throws Exception {
    QueryTestTableHelper qtth = new QueryTestTableHelper(ShardRange.class.getName(), log, teardown, interrupt);
    Connector connector = qtth.connector;
    VisibilityWiseGuysIngestWithModel.writeItAll(connector, VisibilityWiseGuysIngestWithModel.WhatKindaRange.SHARD);
    PrintUtility.printTable(connector, auths, SHARD_TABLE_NAME);
    PrintUtility.printTable(connector, auths, SHARD_INDEX_TABLE_NAME);
    PrintUtility.printTable(connector, auths, MODEL_TABLE_NAME);
    return super.runTestQueryWithGrouping(expected, querystr, startDate, endDate, extraParms, connector);
}
 

private void retrieveAccumuloAuthorizations() throws Exception {
    Map<String,String> trackingMap = accumuloConnectionFactory.getTrackingMap(Thread.currentThread().getStackTrace());
    Connector c = accumuloConnectionFactory.getConnection(AccumuloConnectionFactory.Priority.ADMIN, trackingMap);
    try {
        Authorizations auths = c.securityOperations().getUserAuthorizations(c.whoami());
        HashSet<String> authSet = new HashSet<>();
        for (byte[] auth : auths.getAuthorizations()) {
            authSet.add(new String(auth).intern());
        }
        accumuloUserAuths = Collections.unmodifiableSet(authSet);
        log.debug("Accumulo User Authorizations: {}", accumuloUserAuths);
    } finally {
        accumuloConnectionFactory.returnConnection(c);
    }
}
 

public static Scanner createScanner(Connector connector, String tableName, Collection<Authorizations> authorizations, Query query)
                throws TableNotFoundException {
    Scanner scanner = ScannerHelper.createScanner(connector, tableName, authorizations);
    
    scanner.addScanIterator(getQueryInfoIterator(query, false));
    
    return scanner;
}
 

private void tableCheck(Connector c) throws AccumuloException, AccumuloSecurityException, TableExistsException {
    if (!c.tableOperations().exists(TABLE_NAME)) {
        c.tableOperations().create(TABLE_NAME);
        try {
            IteratorSetting iteratorCfg = new IteratorSetting(19, "ageoff", QueriesTableAgeOffIterator.class);
            c.tableOperations().attachIterator(TABLE_NAME, iteratorCfg, EnumSet.allOf(IteratorScope.class));
        } catch (TableNotFoundException e) {
            throw new AccumuloException("We just created " + TABLE_NAME + " so this shouldn't have happened!", e);
        }
    }
}
 

@Before
public void setupCacheBean() throws IllegalAccessException, SecurityException, NoSuchMethodException {
    qlCache = new CreatedQueryLogicCacheBean();
    queryLogic = PowerMock.createMock(QueryLogic.class);
    conn = PowerMock.createMock(Connector.class);
    internalCache = new ConcurrentHashMap<>();
    
    PowerMock.field(CreatedQueryLogicCacheBean.class, "cache").set(qlCache, internalCache);
    
    PowerMock.mockStatic(System.class, System.class.getMethod("currentTimeMillis"));
}
 

private static Connector getConnector() {
    try {
        return new InMemoryInstance().getConnector("root", new PasswordToken(""));
    } catch (AccumuloException | AccumuloSecurityException e) {
        throw new RuntimeException(e);
    }
}
 

@Test
public void testConstructorWithNullConnector() throws Exception {
    Connector connector = null;
    DatawaveUser user = new DatawaveUser(userDN, UserType.USER, null, null, null, 0L);
    DatawavePrincipal principal = new DatawavePrincipal(Collections.singletonList(user));
    
    RunningQuery query = new RunningQuery(connector, connectionPriority, logic, settings, methodAuths, principal, new QueryMetricFactoryImpl());
    
    assertEquals(connector, query.getConnection());
}
 

@Override
public GenericQueryConfiguration initialize(final Connector connection, final Query settings, final Set<Authorizations> auths) throws Exception {
    // Initialize the config and scanner factory
    final ContentQueryConfiguration config = new ContentQueryConfiguration(this, settings);
    this.scannerFactory = new ScannerFactory(connection);
    config.setConnector(connection);
    config.setAuthorizations(auths);
    
    // Re-assign the view name if specified via params
    Parameter p = settings.findParameter(QueryParameters.CONTENT_VIEW_NAME);
    if (null != p && !StringUtils.isEmpty(p.getParameterValue())) {
        this.viewName = p.getParameterValue();
    }
    
    // Decide whether or not to include the content of child events
    String end;
    p = settings.findParameter(QueryParameters.CONTENT_VIEW_ALL);
    if ((null != p) && (null != p.getParameterValue()) && StringUtils.isNotBlank(p.getParameterValue())) {
        end = ALL;
    } else {
        end = PARENT_ONLY;
    }
    
    // Configure ranges
    final Collection<Range> ranges = this.createRanges(settings, end);
    config.setRanges(ranges);
    
    return config;
}
 

/**
 * Run the demo.
 */
public void execute(
        MiniAccumuloCluster accumulo,
        Connector accumuloConn,
        String ryaTablePrefix,
        RyaSailRepository ryaRepo,
        RepositoryConnection ryaConn,
        MiniFluo fluo,
        FluoClient fluoClient) throws DemoExecutionException;
 

@Before
public void init() throws Exception {
    conf = getConf();
    Connector conn = ConfigUtils.getConnector(conf);
    dao = new AccumuloRyaDAO();
    dao.setConnector(conn);
    dao.init();
}
 

public RfileScanner(Connector connector, Configuration conf, String table, Set<Authorizations> auths, int numQueryThreads) {
    ArgumentChecker.notNull(connector, conf, table, auths);
    this.table = table;
    this.auths = auths;
    this.connector = connector;
    ranges = null;
    authIter = AuthorizationsUtil.minimize(auths).iterator();
    recordIterAuthString = authIter.next().toString();
    iterators = Lists.newArrayList();
    iterators = Collections.synchronizedList(iterators);
    setConfiguration(conf);
}