org.apache.accumulo.core.client.TableNotFoundException Java Examples

@Override
public void configure(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    
    switch (this.tableType) {
        case SHARD:
            configureShardTable(tops);
            break;
        case GIDX:
            configureGidxTable(tops);
            break;
        case GRIDX:
            configureGridxTable(tops);
            break;
        
        case DINDX:
            configureDictionaryTable(tops);
            
            break;
        default:
            // Technically, this is dead code. If 'Configure' is called prior to 'Setup'
            // tableType is null and throws a NullPointerException in the switch statement.
            // If 'Setup' successfully runs to completion then tableType is assigned one
            // of the three other values.
            throw new TableNotFoundException(null, tableName, "Table is not a Shard Type Table");
    }
}
 

private String seedMetadataTable(Connector connector, long time, int keyVersionNum) throws TableNotFoundException, MutationsRejectedException {
    Value emptyVal = new Value();
    SimpleDateFormat dateFormat = new SimpleDateFormat(DateNormalizer.ISO_8601_FORMAT_STRING);
    String dateString = dateFormat.format(new Date(time));
    try (BatchWriter recordWriter = connector.createBatchWriter(metadataTableName, new BatchWriterConfig())) {
        String normalizedVersionNum = NumericalEncoder.encode(Integer.toString(keyVersionNum));
        String rowID = "edge_key";
        String columnFamily = "version";
        String columnQualifier = normalizedVersionNum + "/" + dateString;
        
        Mutation m = new Mutation(rowID);
        
        m.put(new Text(columnFamily), new Text(columnQualifier), emptyVal);
        
        recordWriter.addMutation(m);
    }
    return dateString;
}
 

/**
 * Creates the tables that are needed to load data using this ingest job if they don't already exist. If a table is created, it is configured with the
 * appropriate iterators, aggregators, and locality groups that are required for ingest and query functionality to work correctly.
 *
 * @param tableNames
 *            the names of the table to create if they don't exist
 * @param tops
 *            accumulo table operations helper for checking/creating tables
 * @param conf
 *            the Hadoop {@link Configuration} for retrieving table configuration information
 * @param log
 *            a logger for diagnostic messages
 * @param enableBloomFilters
 *            an indication of whether bloom filters should be enabled in the configuration
 * @throws AccumuloSecurityException
 * @throws AccumuloException
 * @throws TableNotFoundException
 */
protected void createAndConfigureTablesIfNecessary(String[] tableNames, TableOperations tops, NamespaceOperations namespaceOperations, Configuration conf,
                Logger log, boolean enableBloomFilters) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
    for (String table : tableNames) {
        createNamespaceIfNecessary(namespaceOperations, table);
        // If the tables don't exist, then create them.
        try {
            if (!tops.exists(table)) {
                tops.create(table);
            }
        } catch (TableExistsException te) {
            // in this case, somebody else must have created the table after our existence check
            log.info("Tried to create " + table + " but somebody beat us to the punch");
        }
    }
    
    // Pass along the enabling of bloom filters using the configuration
    conf.setBoolean(ShardTableConfigHelper.ENABLE_BLOOM_FILTERS, enableBloomFilters);
    
    configureTablesIfNecessary(tableNames, tops, conf, log);
}
 

@Override
public void configure(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    // add the SummingCombiner Iterator for each iterator scope (majc, minc, scan)
    for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) {
        final StringBuilder propName = new StringBuilder(String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "sum"));
        setPropertyIfNecessary(mTableName, propName.toString(), "19,org.apache.accumulo.core.iterators.user.SummingCombiner", tops, mLog);
        propName.append(".opt.");
        setPropertyIfNecessary(mTableName, propName + "all", "true", tops, mLog);
        setPropertyIfNecessary(mTableName, propName + "lossy", "FALSE", tops, mLog);
        setPropertyIfNecessary(mTableName, propName + "type", "STRING", tops, mLog);
    }
    
    // enable bloom filters if necessary.
    if (mEnableBloomFilters) {
        setPropertyIfNecessary(mTableName, Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ShardIndexKeyFunctor.class.getName(), tops, mLog);
    }
    setPropertyIfNecessary(mTableName, Property.TABLE_BLOOM_ENABLED.getKey(), Boolean.toString(mEnableBloomFilters), tops, mLog);
}
 

/**
 * updates the splits file if the splits in the new file have not decreased beyond the maximum deviation allowed
 */
public void update() {
    try {
        FileSystem fs = FileSystem.get(this.splitsPath.toUri(), conf);
        initAccumuloHelper();
        Path tmpSplitsFile = createTempFile(fs);
        Map<String,Integer> tmpSplitsPerTable = writeSplits(fs, tmpSplitsFile);
        if (null == getFileStatus() || !exceedsMaxSplitsDeviation(tmpSplitsPerTable)) {
            log.info("updating splits file");
            createCacheFile(fs, tmpSplitsFile);
        } else {
            log.info("Deleting " + tmpSplitsFile);
            fs.delete(tmpSplitsFile, false);
        }
    } catch (IOException | AccumuloException | AccumuloSecurityException | TableNotFoundException ex) {
        log.error("Unable to update the splits file", ex);
    }
    
}
 

@Override
public void clear() {
    log.trace("Clearing Accumulo cache for table {}.", tableName);
    try {
        BatchWriterConfig bwCfg = new BatchWriterConfig();
        BatchDeleter deleter = connector.createBatchDeleter(tableName, authorizations, 10, bwCfg);
        try {
            deleter.setRanges(Collections.singletonList(new Range()));
            deleter.delete();
        } finally {
            deleter.close();
        }
    } catch (MutationsRejectedException | TableNotFoundException e) {
        throw new PersistenceException("Unable to clear Accumulo cache for " + tableName, e);
    }
}
 

private boolean isNonEvent(JexlNode node) {
    if (this.nonEventFields == null) {
        try {
            this.nonEventFields = helper.getNonEventFields(config.getDatatypeFilter());
        } catch (TableNotFoundException e) {
            log.error("Could not determine whether field is non event", e);
            throw new RuntimeException("got exception when using MetadataHelper to get non event fields", e);
        }
    }
    List<ASTIdentifier> identifiers = JexlASTHelper.getIdentifiers(node);
    for (ASTIdentifier identifier : identifiers) {
        String deconstructed = JexlASTHelper.deconstructIdentifier(identifier);
        if (deconstructed.equals(Constants.NO_FIELD)) {
            // no field should not be factored in here
            continue;
        }
        if (this.nonEventFields.contains(deconstructed)) {
            return true;
        }
    }
    return false;
}
 

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    
    Configuration conf = context.getConfiguration();
    String user = conf.get(MetricsConfig.USER);
    String password = conf.get(MetricsConfig.PASS);
    String instance = conf.get(MetricsConfig.INSTANCE);
    String zookeepers = conf.get(MetricsConfig.ZOOKEEPERS);
    
    useHourlyPrecision = HourlyPrecisionHelper.checkForHourlyPrecisionOption(context.getConfiguration(), log);
    
    try {
        ZooKeeperInstance inst = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zookeepers));
        Connector con = inst.getConnector(user, new PasswordToken(password));
        ingestScanner = con.createScanner(conf.get(MetricsConfig.INGEST_TABLE, MetricsConfig.DEFAULT_INGEST_TABLE), Authorizations.EMPTY);
    } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) {
        throw new IOException(e);
    }
}
 

public static BatchScanner createBatchScanner(ShardQueryConfiguration config, ScannerFactory scannerFactory, QueryData qd) throws TableNotFoundException {
    final BatchScanner bs = scannerFactory.newScanner(config.getShardTableName(), config.getAuthorizations(), config.getNumQueryThreads(),
                    config.getQuery());
    
    if (log.isTraceEnabled()) {
        log.trace("Running with " + config.getAuthorizations() + " and " + config.getNumQueryThreads() + " threads: " + qd);
    }
    
    bs.setRanges(qd.getRanges());
    
    for (IteratorSetting cfg : qd.getSettings()) {
        bs.addScanIterator(cfg);
    }
    
    return bs;
}
 

/**
 * Set the locality group configuration for a table if necessary. If the specified configuration is not already included in the current group configuration,
 * then the new locality groups are merged with the current set and the locality groups are reset for the table.
 * 
 * @param tableName
 * @param newLocalityGroups
 * @param tops
 * @param log
 * @throws AccumuloException
 * @throws TableNotFoundException
 * @throws AccumuloSecurityException
 */
protected void setLocalityGroupConfigurationIfNecessary(String tableName, Map<String,Set<Text>> newLocalityGroups, TableOperations tops, Logger log)
                throws AccumuloException, TableNotFoundException, AccumuloSecurityException {
    if (areLocalityGroupsConfigured(tableName, newLocalityGroups, tops)) {
        log.debug("Verified the following locality groups are configured for " + tableName + ": " + newLocalityGroups);
        return;
    }
    
    log.info("Creating the locality groups for " + tableName + ": " + newLocalityGroups);
    Map<String,Set<Text>> localityGroups = tops.getLocalityGroups(tableName);
    for (Map.Entry<String,Set<Text>> entry : newLocalityGroups.entrySet()) {
        Set<Text> families = localityGroups.get(entry.getKey());
        if (families == null) {
            families = new HashSet<>();
            localityGroups.put(entry.getKey(), families);
        }
        families.addAll(entry.getValue());
    }
    tops.setLocalityGroups(tableName, localityGroups);
    log.info("Reset the locality groups for " + tableName + " to " + localityGroups);
}
 

/**
 * Create an IndexLookup task to find field names give a JexlNode and a set of Types for that node
 *
 * @param node
 * @param expansionFields
 * @param dataTypes
 * @param ingestDataTypes
 * @param helperRef
 * @return The index lookup instance
 * @throws TableNotFoundException
 */
public static IndexLookup normalizeQueryTerm(JexlNode node, Set<String> expansionFields, Set<Type<?>> dataTypes, Set<String> ingestDataTypes,
                MetadataHelper helperRef) throws TableNotFoundException {
    if (node instanceof ASTEQNode) {
        return normalizeQueryTerm((ASTEQNode) node, expansionFields, dataTypes, ingestDataTypes, helperRef);
    } else if (node instanceof ASTNENode) {
        return normalizeQueryTerm((ASTNENode) node, expansionFields, dataTypes, ingestDataTypes, helperRef);
    } else if (node instanceof ASTERNode) {
        return expandRegexFieldName((ASTERNode) node, expansionFields, dataTypes, ingestDataTypes, helperRef);
    } else if (node instanceof ASTNRNode) {
        return expandRegexFieldName((ASTNRNode) node, expansionFields, dataTypes, ingestDataTypes, helperRef);
    } else if (node instanceof ASTLENode) {
        throw new UnsupportedOperationException("Cannot expand an unbounded range");
    } else if (node instanceof ASTLTNode) {
        throw new UnsupportedOperationException("Cannot expand an unbounded range");
    } else if (node instanceof ASTGENode) {
        throw new UnsupportedOperationException("Cannot expand an unbounded range");
    } else if (node instanceof ASTGTNode) {
        throw new UnsupportedOperationException("Cannot expand an unbounded range");
    } else {
        return new EmptyIndexLookup();
    }
}
 

@Override
public QueryModel getQueryModel() {
    QueryModel queryModel = null;
    if (config.getQueryModel() != null) {
        log.debug("Using a cached query model");
        queryModel = config.getQueryModel();
    } else if (null != config.getModelName() && null != config.getModelTableName()) {
        log.debug("Generating a query model");
        try {
            queryModel = metadataHelper.getQueryModel(config.getModelTableName(), config.getModelName(), config.getUnevaluatedFields(),
                            config.getDatatypeFilter());
            config.setQueryModel(queryModel);
        } catch (TableNotFoundException e) {
            QueryException qe = new QueryException(DatawaveErrorCode.QUERY_MODEL_FETCH_ERROR, e);
            log.error(qe);
            throw new DatawaveFatalQueryException(qe);
        }
        
        if (log.isTraceEnabled()) {
            log.trace("forward queryModel: " + queryModel.getForwardQueryMapping());
            log.trace("reverse queryModel: " + queryModel.getReverseQueryMapping());
        }
    }
    return queryModel;
}
 

public RangeStream(ShardQueryConfiguration config, ScannerFactory scanners, MetadataHelper metadataHelper) {
    this.config = config;
    this.scanners = scanners;
    this.metadataHelper = metadataHelper;
    int maxLookup = (int) Math.max(config.getNumIndexLookupThreads(), 1);
    executor = Executors.newFixedThreadPool(maxLookup);
    runnables = new LinkedBlockingDeque<>();
    int executeLookupMin = (int) Math.max(maxLookup / 2, 1);
    streamExecutor = new ThreadPoolExecutor(executeLookupMin, maxLookup, 100, TimeUnit.MILLISECONDS, runnables);
    fieldDataTypes = config.getQueryFieldsDatatypes();
    collapseUids = config.getCollapseUids();
    try {
        Set<String> ioFields = metadataHelper.getIndexOnlyFields(null);
        if (null != ioFields) {
            indexOnlyFields.addAll(ioFields);
        }
    } catch (TableNotFoundException e) {
        // ignore
    }
}
 

protected void addMetadataEntries(Multimap<String,Key> metadataEntries) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
    MultiTableBatchWriter multiTableWriter = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    BatchWriter writer = multiTableWriter.getBatchWriter(QueryTestTableHelper.METADATA_TABLE_NAME);
    for (String field : metadataEntries.keySet()) {
        Mutation mutation = new Mutation(new Text(field));
        for (Key key : metadataEntries.get(field)) {
            metadataEntries.put(field, key);
            mutation.put(key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibilityParsed(), new Value());
        }
        writer.addMutation(mutation);
    }
    writer.close();
    connector.tableOperations().compact(QueryTestTableHelper.METADATA_TABLE_NAME, new Text("\0"), new Text("~"), true, true);
}
 

public static BatchScanner createBatchScanner(Connector connector, String tableName, Collection<Authorizations> authorizations, int numQueryThreads,
                Query query, boolean reportErrors) throws TableNotFoundException {
    BatchScanner batchScanner = ScannerHelper.createBatchScanner(connector, tableName, authorizations, numQueryThreads);
    
    batchScanner.addScanIterator(getQueryInfoIterator(query, reportErrors));
    
    return batchScanner;
}
 

public synchronized BatchScanner newScanner(String tableName, Set<Authorizations> auths, int threads, Query query) throws TableNotFoundException {
    if (open) {
        BatchScanner bs = QueryScannerHelper.createBatchScanner(cxn, tableName, auths, threads, query);
        log.debug("Created scanner " + System.identityHashCode(bs));
        if (log.isTraceEnabled()) {
            log.trace("Adding instance " + bs.hashCode());
        }
        instances.add(bs);
        return bs;
    } else {
        throw new IllegalStateException("Factory has been locked. No new scanners can be created.");
    }
}
 

/**
 * Initializes the scanner resource
 * 
 * @param auths
 * @param tableName
 * @throws TableNotFoundException
 * 
 */
@Override
protected void init(final String tableName, final Set<Authorizations> auths, Collection<Range> currentRange) throws TableNotFoundException {
    Preconditions.checkNotNull(tableName);
    Preconditions.checkArgument(null != currentRange && !currentRange.isEmpty());
    
    // copy the appropriate variables.
    ranges = Lists.newArrayList(currentRange);
    
    this.tableName = tableName;
    
    this.auths = Sets.newHashSet(auths);
    
    if (log.isTraceEnabled())
        log.trace("Creating scanner resource from " + tableName + " " + auths + " " + currentRange);
    
    internalTimer = new StopWatch();
    internalTimer.start();
    
    // let's pre-compute the hashcode.
    hashCode += new HashCodeBuilder().append(tableName).append(auths).append(ranges).toHashCode();
    
    baseScanner = ScannerHelper.createBatchScanner(getConnector(), tableName, auths, 2);
    
    if (baseScanner != null) {
        ((BatchScanner) baseScanner).setRanges(currentRange);
    }
    
}
 

@Test
public void testIndexedNeNull() throws ParseException, TableNotFoundException {
    ASTJexlScript query = JexlASTHelper.parseJexlQuery("INDEXEDFIELD != null");
    
    HashSet indexedFields = new HashSet();
    indexedFields.add("INDEXONLYFIELD");
    indexedFields.add("INDEXEDFIELD");
    
    HashSet<String> indexOnlyFields = new HashSet<>();
    indexOnlyFields.add("INDEXONLYFIELD");
    
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    EasyMock.expect(helper.getIndexOnlyFields(null)).andReturn(indexOnlyFields).anyTimes();
    
    replayAll();
    
    boolean executable = ExecutableDeterminationVisitor.isExecutable(query, config, helper);
    Assert.assertFalse(executable);
    
    boolean fiExecutable = ExecutableDeterminationVisitor.isExecutable(query, config, helper, true);
    Assert.assertFalse(fiExecutable);
    
    ExecutableDeterminationVisitor.STATE state = ExecutableDeterminationVisitor.getState(query, config, helper);
    Assert.assertEquals(ExecutableDeterminationVisitor.STATE.NON_EXECUTABLE, state);
    
    ExecutableDeterminationVisitor.STATE fiState = ExecutableDeterminationVisitor.getState(query, config, helper, true);
    Assert.assertEquals(ExecutableDeterminationVisitor.STATE.NON_EXECUTABLE, fiState);
    
    verifyAll();
}
 

protected Map<String,Map<TabletId,List<Range>>> binRanges(List<Range> ranges) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    Map<String,Map<TabletId,List<Range>>> binnedRanges = new HashMap<>();
    Locations locations = config.getConnector().tableOperations().locate(tableName, ranges);
    Map<TabletId,List<Range>> tabletToRange = locations.groupByTablet();
    for (TabletId tid : tabletToRange.keySet()) {
        binnedRanges.put(locations.getTabletLocation(tid), tabletToRange);
    }
    
    // truncate the ranges to within the tablets... this makes it easier
    // to know what work
    // needs to be redone when failures occurs and tablets have merged
    // or split
    Map<String,Map<TabletId,List<Range>>> binnedRanges2 = new HashMap<>();
    for (Entry<String,Map<TabletId,List<Range>>> entry : binnedRanges.entrySet()) {
        Map<TabletId,List<Range>> tabletMap = new HashMap<>();
        binnedRanges2.put(entry.getKey(), tabletMap);
        for (Entry<TabletId,List<Range>> tabletRanges : entry.getValue().entrySet()) {
            Range tabletRange = tabletRanges.getKey().toRange();
            List<Range> clippedRanges = new ArrayList<>();
            tabletMap.put(tabletRanges.getKey(), clippedRanges);
            for (Range range : tabletRanges.getValue())
                clippedRanges.add(tabletRange.clip(range));
        }
    }
    
    binnedRanges.clear();
    binnedRanges.putAll(binnedRanges2);
    
    return binnedRanges;
}
 

public static BatchScanner configureBatchScannerForDiscovery(DiscoveryQueryConfiguration config, ScannerFactory scannerFactory, String tableName,
                Collection<Range> seekRanges, Set<Text> columnFamilies, Multimap<String,String> literals, Multimap<String,String> patterns,
                Multimap<String,LiteralRange<String>> ranges, boolean reverseIndex) throws TableNotFoundException {
    
    // if we have no ranges, then nothing to scan
    if (seekRanges.isEmpty()) {
        return null;
    }
    
    BatchScanner bs = scannerFactory.newScanner(tableName, config.getAuthorizations(), config.getNumQueryThreads(), config.getQuery());
    bs.setRanges(seekRanges);
    if (!columnFamilies.isEmpty()) {
        for (Text family : columnFamilies) {
            bs.fetchColumnFamily(family);
        }
    }
    
    // The begin date from the query may be down to the second, for doing lookups in the index we want to use the day because
    // the times in the index table have been truncated to the day.
    Date begin = DateUtils.truncate(config.getBeginDate(), Calendar.DAY_OF_MONTH);
    // we don't need to bump up the end date any more because it's not apart of the range set on the scanner
    Date end = config.getEndDate();
    
    LongRange dateRange = new LongRange(begin.getTime(), end.getTime());
    
    ShardIndexQueryTableStaticMethods.configureGlobalIndexDateRangeFilter(config, bs, dateRange);
    ShardIndexQueryTableStaticMethods.configureGlobalIndexDataTypeFilter(config, bs, config.getDatatypeFilter());
    
    configureIndexMatchingIterator(config, bs, literals, patterns, ranges, reverseIndex);
    
    IteratorSetting discoveryIteratorSetting = new IteratorSetting(config.getBaseIteratorPriority() + 50, DiscoveryIterator.class);
    discoveryIteratorSetting.addOption(REVERSE_INDEX, Boolean.toString(reverseIndex));
    discoveryIteratorSetting.addOption(SEPARATE_COUNTS_BY_COLVIS, config.getSeparateCountsByColVis().toString());
    if (config.getShowReferenceCount()) {
        discoveryIteratorSetting.addOption(SHOW_REFERENCE_COUNT, config.getShowReferenceCount().toString());
    }
    bs.addScanIterator(discoveryIteratorSetting);
    
    return bs;
}
 

protected void redistributeQueries(Multimap<String,QueryPlan> serverPlan, QueryPlan currentPlan) throws AccumuloException, AccumuloSecurityException,
                TableNotFoundException {
    
    List<Range> ranges = Lists.newArrayList(currentPlan.getRanges());
    if (!ranges.isEmpty()) {
        Map<String,Map<TabletId,List<Range>>> binnedRanges = binRanges(ranges);
        
        for (String server : binnedRanges.keySet()) {
            Map<TabletId,List<Range>> hostedExtentMap = binnedRanges.get(server);
            
            Iterable<Range> rangeIter = Lists.newArrayList();
            
            for (Entry<TabletId,List<Range>> rangeEntry : hostedExtentMap.entrySet()) {
                if (log.isTraceEnabled())
                    log.trace("Adding range from " + rangeEntry.getValue());
                rangeIter = Iterables.concat(rangeIter, rangeEntry.getValue());
            }
            
            if (log.isTraceEnabled())
                log.trace("Adding query tree " + JexlStringBuildingVisitor.buildQuery(currentPlan.getQueryTree()) + " " + currentPlan.getSettings().size()
                                + " for " + server);
            
            serverPlan.put(server, new QueryPlan(currentPlan.getQueryTree(), rangeIter, currentPlan.getSettings(), currentPlan.getColumnFamilies()));
            
        }
    }
    
}
 

private void writeShardIndexKeys(BatchWriterConfig bwConfig, final List<Map.Entry<Multimap<String,String>,UID>> data, String table, boolean reverse)
                throws MutationsRejectedException, TableNotFoundException {
    Map<String,RawMetaData> meta = this.cfgData.getMetadata();
    Set<String> fields;
    if (reverse) {
        fields = this.fieldConfig.getReverseIndexFields();
    } else {
        fields = this.fieldConfig.getIndexFields();
    }
    try (BatchWriter bw = this.conn.createBatchWriter(table, bwConfig)) {
        for (Map.Entry<Multimap<String,String>,UID> rawEntries : data) {
            UID uid = rawEntries.getValue();
            Multimap<String,String> rawData = rawEntries.getKey();
            String shardId = extractShard(rawData);
            long timestamp = shardDateToMillis(shardId);
            shardId = shardId + "_0";
            
            for (Map.Entry<String,String> entry : rawData.entries()) {
                if (fields.contains(entry.getKey())) {
                    Normalizer<?> norm = meta.get(entry.getKey().toLowerCase()).normalizer;
                    String normVal = norm.normalize(entry.getValue());
                    if (reverse) {
                        normVal = new StringBuilder(normVal).reverse().toString();
                    }
                    Mutation mut = new Mutation(normVal);
                    Uid.List.Builder builder = Uid.List.newBuilder();
                    builder.addUID(uid.toString());
                    builder.setCOUNT(1);
                    builder.setIGNORE(false);
                    mut.put(entry.getKey().toUpperCase(), shardId + NULL_SEP + this.dataType, this.cfgData.getDefaultVisibility(), timestamp, new Value(
                                    builder.build().toByteArray()));
                    bw.addMutation(mut);
                }
            }
        }
    }
}
 

@Override
protected void setLocalityGroupConfigurationIfNecessary(String tableName, Map<String,Set<Text>> newLocalityGroups, TableOperations tops, Logger log)
                throws AccumuloException, TableNotFoundException, AccumuloSecurityException {
    
    setLocalityGroupConfigurationCalled = true;
    
    if (!overrideSetLocalityGroupConfigured) {
        
        super.setLocalityGroupConfigurationIfNecessary(tableName, newLocalityGroups, tops, log);
    }
}
 

/**
 * Loads test data from a CSV file.
 * 
 * @param file
 *            uri of CSV file
 */
public void loadGroupingData(final URI file) {
    Assert.assertFalse("datatype has already been configured(" + this.datatype + ")", this.rawData.containsKey(this.datatype));
    
    try (final Reader reader = Files.newBufferedReader(Paths.get(file)); final CSVReader csv = new CSVReader(reader)) {
        GroupingAccumuloWriter writer = new GroupingAccumuloWriter(this.datatype, this.accumuloConn, this.cfgData.getDateField(), this.fieldIndex,
                        this.cfgData);
        
        Set<RawData> bookData = new HashSet<>();
        List<Map.Entry<Multimap<String,String>,UID>> loadData = new ArrayList<>();
        
        int count = 0;
        String[] data;
        while (null != (data = csv.readNext())) {
            RawData rawEntry = new BooksRawData(this.datatype, this.getHeaders(), this.metadata, data);
            bookData.add(rawEntry);
            
            final Multimap<String,String> mm = ((BooksRawData) rawEntry).toMultimap();
            UID uid = UID.builder().newId(mm.toString().getBytes(), "");
            loadData.add(Maps.immutableEntry(mm, uid));
            count++;
        }
        this.rawData.put(this.datatype, bookData);
        writer.addData(loadData);
    } catch (IOException | MutationsRejectedException | TableNotFoundException e) {
        throw new AssertionError(e);
    }
}
 

@Override
public void configure(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    super.configure(tops);
    for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) {
        configureToDropBadData(tops, scope.name());
    }
}
 

/**
 * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}.
 * 
 * @param conf
 *            the Hadoop configuration object
 * @throws IOException
 *             if the configuration is improperly configured
 */
protected static void validateOptions(Configuration conf) throws IOException {
    if (!conf.getBoolean(INPUT_INFO_HAS_BEEN_SET, false))
        throw new IOException("Input info has not been set.");
    if (!conf.getBoolean(INSTANCE_HAS_BEEN_SET, false))
        throw new IOException("Instance info has not been set.");
    
    /*
     * if (conf.get(RACKSTRATEGY) == null) { throw new IOException("Rack strategy must be set."); }
     */
    // validate that we can connect as configured
    try {
        Connector c = getInstance(conf).getConnector(getUsername(conf), new PasswordToken(getPassword(conf)));
        if (!c.securityOperations().authenticateUser(getUsername(conf), new PasswordToken(getPassword(conf))))
            throw new IOException("Unable to authenticate user");
        if (!c.securityOperations().hasTablePermission(getUsername(conf), getTablename(conf), TablePermission.READ))
            throw new IOException("Unable to access table");
        
        if (!usesLocalIterators(conf)) {
            // validate that any scan-time iterators can be loaded by the the tablet servers
            for (AccumuloIterator iter : getIterators(conf)) {
                if (!c.tableOperations().testClassLoad(getTablename(conf), iter.getIteratorClass(), SortedKeyValueIterator.class.getName())
                                && !c.instanceOperations().testClassLoad(iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
                    throw new AccumuloException("Servers are unable to load " + iter.getIteratorClass() + " as a " + SortedKeyValueIterator.class.getName());
            }
        }
        
    } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
        throw new IOException(e);
    }
}
 

private String setFrequencyCombiner(TableOperations tops, String scopeName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scopeName, "FrequencyCombiner");
    setPropertyIfNecessary(tableName, stem, "10," + SummingCombiner.class.getName(), tops, log);
    setPropertyIfNecessary(tableName, stem + ".opt.columns", ColumnFamilyConstants.COLF_F.toString(), tops, log);
    setPropertyIfNecessary(tableName, stem + ".opt.type", "VARLEN", tops, log);
    return stem;
}
 

public FieldIndexIterable(Connector con, String shardTable, String eventUid, String datatype, Set<Authorizations> userAuths, List<Range> ranges)
                throws TableNotFoundException {
    scanner = ScannerHelper.createBatchScanner(con, shardTable, userAuths, ranges.size());
    scanner.setRanges(ranges);
    Map<String,String> options = new HashMap();
    options.put(FieldIndexDocumentFilter.DATA_TYPE_OPT, datatype);
    options.put(FieldIndexDocumentFilter.EVENT_UID_OPT, eventUid);
    IteratorSetting settings = new IteratorSetting(100, FieldIndexDocumentFilter.class, options);
    scanner.addScanIterator(settings);
}
 

@Override
public void configure(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    if (tableName != null) {
        for (IteratorScope scope : IteratorScope.values()) {
            setFrequencyCombiner(tops, scope.name());
            setCombinerForCountMetadata(tops, scope.name());
            setCombinerForEdgeMetadata(tops, scope.name());
        }
    }
    
}
 

@Override
public boolean contains(Object key) {
    try (Scanner scanner = connector.createScanner(tableName, authorizations)) {
        scanner.setRange(new Range(String.valueOf(key)));
        Iterator<Map.Entry<Key,Value>> iterator = scanner.iterator();
        return iterator.hasNext();
    } catch (TableNotFoundException e) {
        throw new PersistenceException(e);
    }
}