org.apache.accumulo.core.client.BatchWriter Java Examples

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    tops.create(TableName.DATE_INDEX);
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

private static void cleanupAndCreateTables(Properties props) throws Exception {
  FileSystem hdfs = FileSystem.get(new Configuration());
  if (hdfs.exists(rootPath)) {
    hdfs.delete(rootPath, true);
  }
  try (AccumuloClient client = Accumulo.newClient().from(props).build()) {
    if (client.tableOperations().exists(inputTable)) {
      client.tableOperations().delete(inputTable);
    }
    if (client.tableOperations().exists(outputTable)) {
      client.tableOperations().delete(outputTable);
    }
    // Create tables
    client.tableOperations().create(inputTable);
    client.tableOperations().create(outputTable);

    // Write data to input table
    try (BatchWriter bw = client.createBatchWriter(inputTable)) {
      for (int i = 0; i < 100; i++) {
        Mutation m = new Mutation(String.format("%03d", i));
        m.at().family("cf1").qualifier("cq1").put("" + i);
        bw.addMutation(m);
      }
    }
  }
}
 

public static void main(String[] args) throws Exception {
  Opts opts = new Opts();
  BatchWriterOpts bwOpts = new BatchWriterOpts();
  opts.parseArgs(FileDataIngest.class.getName(), args, bwOpts);

  try (AccumuloClient client = opts.createAccumuloClient()) {
    if (!client.tableOperations().exists(opts.getTableName())) {
      client.tableOperations().create(opts.getTableName());
      client.tableOperations().attachIterator(opts.getTableName(),
          new IteratorSetting(1, ChunkCombiner.class));
    }
    try (BatchWriter bw = client.createBatchWriter(opts.getTableName(),
        bwOpts.getBatchWriterConfig())) {
      FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility);
      for (String filename : opts.files) {
        fdi.insertFileData(filename, bw);
      }
    }
  }
}
 

private long ingestRandom(long timestampInitial, int decrementMillis) {
    long ts = timestampInitial;
    Collection<Mutation> mutations = new ArrayList<>(TestMetricWriter.DEFAULT_BATCHES);
    int idx = 0;
    while (idx++ < TestMetricWriter.DEFAULT_BATCHES) {
        ts = ts - decrementMillis;
        Metric m = metrics.randomMetric(ts);
        Mutation mutation = MetricAdapter.toMutation(m);
        mutations.add(mutation);
    }

    try {
        BatchWriterConfig cfg = new BatchWriterConfig().setMaxWriteThreads(4);
        try (BatchWriter writer = connector.createBatchWriter(tableName, cfg)) {
            writer.addMutations(mutations);
        }
        connector.tableOperations().flush(tableName, null, null, true);
    } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
        throw new RuntimeException(e);
    }

    return ts;
}
 

private void deleteRows(BatchWriter writer, List<Key> rowsToDelete, Options options) throws MutationsRejectedException {
    rowsToDelete.sort(Comparator.comparingLong(Key::getTimestamp));
    int i = 0;
    for (Key key : rowsToDelete) {
        if (i < rowsToDelete.size() - options.getVersionsToKeep()) {
            LOGGER.debug("deleting row: %s", key.getRow().toString());
            if (!options.isDryRun()) {
                Mutation mutation = new Mutation(key.getRow());
                mutation.putDelete(
                    key.getColumnFamily(),
                    key.getColumnQualifier(),
                    key.getColumnVisibilityParsed(),
                    key.getTimestamp()
                );
                writer.addMutation(mutation);
            }
        } else {
            if (options.isDryRun()) {
                LOGGER.debug("skipping row: %s", key.getRow().toString());
            }
        }
        i++;
    }
}
 

protected Multimap<String,Key> removeMetadataEntries(Set<String> fields, Text cf) throws AccumuloSecurityException, AccumuloException,
                TableNotFoundException {
    Multimap<String,Key> metadataEntries = HashMultimap.create();
    MultiTableBatchWriter multiTableWriter = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    BatchWriter writer = multiTableWriter.getBatchWriter(QueryTestTableHelper.METADATA_TABLE_NAME);
    for (String field : fields) {
        Mutation mutation = new Mutation(new Text(field));
        Scanner scanner = connector.createScanner(QueryTestTableHelper.METADATA_TABLE_NAME, new Authorizations());
        scanner.fetchColumnFamily(cf);
        scanner.setRange(new Range(new Text(field)));
        boolean foundEntries = false;
        for (Map.Entry<Key,Value> entry : scanner) {
            foundEntries = true;
            metadataEntries.put(field, entry.getKey());
            mutation.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getColumnVisibilityParsed());
        }
        scanner.close();
        if (foundEntries) {
            writer.addMutation(mutation);
        }
    }
    writer.close();
    connector.tableOperations().compact(QueryTestTableHelper.METADATA_TABLE_NAME, new Text("\0"), new Text("~"), true, true);
    return metadataEntries;
}
 

public void testColumns() throws Exception {
    BatchWriter batchWriter = connector.createBatchWriter(table, 10l, 10l, 2);
    Mutation row = new Mutation("a");
    row.put("cf1", "cq", new Value(new byte[0]));
    row.put("cf2", "cq", new Value(new byte[0]));
    row.put("cf3", "cq1", new Value(new byte[0]));
    row.put("cf3", "cq2", new Value(new byte[0]));
    batchWriter.addMutation(row);
    batchWriter.flush();
    batchWriter.close();

    String location = "accumulo://" + table + "?instance=" + instance + "&user=" + user + "&password=" + pwd + "&range=a|c&columns=cf1,cf3|cq1&mock=true";
    AccumuloStorage storage = createAccumuloStorage(location);
    int count = 0;
    while (true) {
        Tuple next = storage.getNext();
        if (next == null)
            break;
        assertEquals(6, next.size());
        count++;
    }
    assertEquals(2, count);
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

@Before
public void setupInstance() throws Exception {
  tableName = getUniqueNames(1)[0];
  client = Accumulo.newClient().from(getClientProperties()).build();
  client.tableOperations().create(tableName);
  BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig());
  ColumnVisibility cv = new ColumnVisibility();
  // / has 1 dir
  // /local has 2 dirs 1 file
  // /local/user1 has 2 files
  bw.addMutation(Ingest.buildMutation(cv, "/local", true, false, true, 272, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/user1", true, false, true, 272, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/user2", true, false, true, 272, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/file", false, false, false, 1024, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/file", false, false, false, 1024, 23456, null));
  bw.addMutation(
      Ingest.buildMutation(cv, "/local/user1/file1", false, false, false, 2024, 12345, null));
  bw.addMutation(
      Ingest.buildMutation(cv, "/local/user1/file2", false, false, false, 1028, 23456, null));
  bw.close();
}
 

private void createEntries(Opts opts) throws TableNotFoundException, AccumuloException {

    // Trace the write operation. Note, unless you flush the BatchWriter, you will not capture
    // the write operation as it is occurs asynchronously. You can optionally create additional
    // Spans
    // within a given Trace as seen below around the flush
    TraceScope scope = Trace.startSpan("Client Write", Sampler.ALWAYS);

    System.out.println("TraceID: " + Long.toHexString(scope.getSpan().getTraceId()));
    try (BatchWriter batchWriter = client.createBatchWriter(opts.getTableName())) {
      Mutation m = new Mutation("row");
      m.put("cf", "cq", "value");

      batchWriter.addMutation(m);
      // You can add timeline annotations to Spans which will be able to be viewed in the Monitor
      scope.getSpan().addTimelineAnnotation("Initiating Flush");
      batchWriter.flush();
    }
    scope.close();
  }
 

public static void index(int numPartitions, Text docId, String doc, String splitRegex,
    BatchWriter bw) throws Exception {

  String[] tokens = doc.split(splitRegex);

  Text partition = genPartition(doc.hashCode() % numPartitions);

  Mutation m = new Mutation(partition);

  HashSet<String> tokensSeen = new HashSet<>();

  for (String token : tokens) {
    token = token.toLowerCase();

    if (!tokensSeen.contains(token)) {
      tokensSeen.add(token);
      m.put(new Text(token), docId, new Value(new byte[0]));
    }
  }

  if (m.size() > 0)
    bw.addMutation(m);
}
 

public static void index(int numPartitions, File src, String splitRegex, BatchWriter bw)
    throws Exception {
  if (src.isDirectory()) {
    File[] files = src.listFiles();
    if (files != null) {
      for (File child : files) {
        index(numPartitions, child, splitRegex, bw);
      }
    }
  } else {
    FileReader fr = new FileReader(src);

    StringBuilder sb = new StringBuilder();

    char data[] = new char[4096];
    int len;
    while ((len = fr.read(data)) != -1) {
      sb.append(data, 0, len);
    }

    fr.close();

    index(numPartitions, new Text(src.getAbsolutePath()), sb.toString(), splitRegex, bw);
  }

}
 

@Test
public void testInfoWithoutChunks() throws Exception {
  client.tableOperations().create(tableName);
  BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig());
  for (Entry<Key,Value> e : baddata) {
    Key k = e.getKey();
    Mutation m = new Mutation(k.getRow());
    m.put(k.getColumnFamily(), k.getColumnQualifier(),
        new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
    bw.addMutation(m);
  }
  bw.close();

  assertEquals(0, CIFTester.main(tableName, CIFTester.TestBadData.class.getName()));
  assertEquals(1, assertionErrors.get(tableName).size());
}
 

/**
 * Add the property to this index.
 * 
 * <p/>Note that this requires a round-trip to Accumulo to see
 * if the property exists if the provided key has an index.
 * So for best performance, create indices after bulk ingest.
 * <p/>If the force parameter is true, set the property regardless
 * of whether indexing is enabled for the given key. This is needed
 * for {@link IndexableGraph} operations.
 * @param element
 * @param key
 * @param value
 * @param force
 */
public void setPropertyForIndex(Element element, String key, Object value,
    boolean force) {
  AccumuloGraphUtils.validateProperty(key, value);
  if (force || globals.getConfig().getAutoIndex() ||
      globals.getIndexMetadataWrapper()
      .getIndexedKeys(elementType).contains(key)) {
    BatchWriter writer = getWriter();

    Object oldValue = element.getProperty(key);
    if (oldValue != null && !oldValue.equals(value)) {
      Mutators.apply(writer, new IndexValueMutator.Delete(element, key, oldValue));
    }

    Mutators.apply(writer, new IndexValueMutator.Add(element, key, value));
    globals.checkedFlush();
  }
}
 

@Test
public void testErrorOnNextWithoutClose() throws Exception {
  client.tableOperations().create(tableName);
  BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig());

  for (Entry<Key,Value> e : data) {
    Key k = e.getKey();
    Mutation m = new Mutation(k.getRow());
    m.put(k.getColumnFamily(), k.getColumnQualifier(),
        new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
    bw.addMutation(m);
  }
  bw.close();

  assertEquals(1, CIFTester.main(tableName, CIFTester.TestNoClose.class.getName()));
  assertEquals(1, assertionErrors.get(tableName).size());
  // this should actually exist, in addition to the dummy entry
  assertEquals(2, assertionErrors.get(tableName + "_map_ioexception").size());
}
 

@Test
public void test() throws Exception {
  client.tableOperations().create(tableName);
  BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig());

  for (Entry<Key,Value> e : data) {
    Key k = e.getKey();
    Mutation m = new Mutation(k.getRow());
    m.put(k.getColumnFamily(), k.getColumnQualifier(),
        new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
    bw.addMutation(m);
  }
  bw.close();

  assertEquals(0, CIFTester.main(tableName, CIFTester.TestMapper.class.getName()));
  assertEquals(1, assertionErrors.get(tableName).size());
}
 

/**
 * Remove an interval instant
 *
 * @param writer
 * @param cv
 * @param instant
 * @throws MutationsRejectedException
 */
public void removeInstant(final BatchWriter writer, final TemporalInstant instant, final Statement statement) throws MutationsRejectedException {
    final KeyParts keyParts = new KeyParts(statement, instant);
    for (final KeyParts  k: keyParts) {
        final Mutation m = new Mutation(k.getStoreKey());
        m.putDelete(k.cf, k.cq);
        writer.addMutation(m);
    }
}
 

@Test
public void testAgeOffIterator() throws Exception {
    InMemoryInstance instance = new InMemoryInstance();
    Connector connector = instance.getConnector("root", new PasswordToken(""));
    
    connector.tableOperations().create(TABLE_NAME);
    IteratorSetting iteratorCfg = new IteratorSetting(19, "ageoff", QueriesTableAgeOffIterator.class);
    connector.tableOperations().attachIterator(TABLE_NAME, iteratorCfg, EnumSet.allOf(IteratorScope.class));
    
    long now = System.currentTimeMillis();
    // Write in a couple of keys with varying timestamps
    BatchWriter writer = connector.createBatchWriter(TABLE_NAME, new BatchWriterConfig().setMaxLatency(30, TimeUnit.MILLISECONDS).setMaxMemory(1024L)
                    .setMaxWriteThreads(1));
    
    Mutation m1 = new Mutation("row1");
    m1.put("colf1", "colq1", now, "");
    writer.addMutation(m1);
    
    Mutation m2 = new Mutation("row2");
    m2.put("colf2", "colq2", (now + 100000), "");
    writer.addMutation(m2);
    
    writer.close();
    
    // Scan the entire table, we should only see keys whose timestamps are greater than or equal to now.
    // Mutation 1 should be expired by now, we should only see Mutation 2;
    boolean sawRow2 = false;
    Scanner scanner = connector.createScanner(TABLE_NAME, new Authorizations());
    for (Entry<Key,Value> entry : scanner) {
        if (entry.getKey().getRow().toString().equals("row1"))
            Assert.fail("We saw row1 when it should be expired.");
        if (entry.getKey().getRow().toString().equals("row2"))
            sawRow2 = true;
    }
    if (!sawRow2)
        Assert.fail("We did not see row2 and we should have");
}
 

protected void insertForwardModel(String from, String to) throws Throwable {
    BatchWriterConfig config = new BatchWriterConfig().setMaxMemory(1024L).setMaxLatency(1, TimeUnit.SECONDS).setMaxWriteThreads(1);
    ColumnVisibility viz = new ColumnVisibility("FOO");
    
    try (BatchWriter writer = connector.createBatchWriter(QueryTestTableHelper.METADATA_TABLE_NAME, config)) {
        Mutation m = new Mutation(from.toUpperCase());
        m.put("DATAWAVE", to.toUpperCase() + "\u0000forward", viz, blank);
        writer.addMutation(m);
    }
}
 

private static void addFiTokens(BatchWriter bw, WhatKindaRange range, String field, String phrase, String uid) throws MutationsRejectedException {
    Mutation fi = new Mutation(shard);
    fi.put("fi\u0000" + field.toUpperCase(), lcNoDiacriticsType.normalize(phrase) + "\u0000" + datatype + "\u0000" + uid, columnVisibility, timeStamp,
                    emptyValue);
    
    String[] tokens = phrase.split(" ");
    for (String token : tokens) {
        fi.put("fi\u0000" + field.toUpperCase(), lcNoDiacriticsType.normalize(token) + "\u0000" + datatype + "\u0000" + uid, columnVisibility, timeStamp,
                        emptyValue);
    }
    bw.addMutation(fi);
}
 

public static void main(String[] args)
    throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
  ClientOpts opts = new ClientOpts();
  opts.parseArgs(Insert.class.getName(), args);

  try (AccumuloClient client = Accumulo.newClient().from(opts.getClientPropsPath()).build()) {
    try {
      client.tableOperations().create("hellotable");
    } catch (TableExistsException e) {
      // ignore
    }

    try (BatchWriter bw = client.createBatchWriter("hellotable")) {
      log.trace("writing ...");
      for (int i = 0; i < 10000; i++) {
        Mutation m = new Mutation(String.format("row_%d", i));
        for (int j = 0; j < 5; j++) {
          m.put("colfam", String.format("colqual_%d", j),
              new Value((String.format("value_%d_%d", i, j)).getBytes()));
        }
        bw.addMutation(m);
        if (i % 100 == 0) {
          log.trace(String.valueOf(i));
        }
      }
    }
  }
}
 

private void persistTestMutations(int numRows, int entriesPerRow) throws TableNotFoundException, MutationsRejectedException {

        BatchWriter writer = connector.createBatchWriter("test", 1000, 1000, 1);

        for (int j = 0; j < numRows; j++) {
            Mutation m = new Mutation(Integer.toString(j));
            for (int i = 0; i < entriesPerRow; i++)
                m.put(new Text(Integer.toString(j)), new Text(String.valueOf(i)), new Value("".getBytes()));

            writer.addMutation(m);
        }
        writer.flush();
    }
 

private static void recurse(File src, ColumnVisibility cv, BatchWriter dirBW, BatchWriter indexBW,
    FileDataIngest fdi, BatchWriter data) throws Exception {
  // ingest this File
  ingest(src, cv, dirBW, indexBW, fdi, data);
  // recurse into subdirectories
  if (src.isDirectory()) {
    File[] files = src.listFiles();
    if (files == null)
      return;
    for (File child : files) {
      recurse(child, cv, dirBW, indexBW, fdi, data);
    }
  }
}
 

public static void main(String[] args) throws Exception {
  Opts opts = new Opts();
  BatchWriterOpts bwOpts = new BatchWriterOpts();
  opts.parseArgs(Ingest.class.getName(), args, bwOpts);

  try (AccumuloClient client = opts.createAccumuloClient()) {
    if (!client.tableOperations().exists(opts.nameTable))
      client.tableOperations().create(opts.nameTable);
    if (!client.tableOperations().exists(opts.indexTable))
      client.tableOperations().create(opts.indexTable);
    if (!client.tableOperations().exists(opts.dataTable)) {
      client.tableOperations().create(opts.dataTable);
      client.tableOperations().attachIterator(opts.dataTable,
          new IteratorSetting(1, ChunkCombiner.class));
    }

    BatchWriter dirBW = client.createBatchWriter(opts.nameTable, bwOpts.getBatchWriterConfig());
    BatchWriter indexBW = client.createBatchWriter(opts.indexTable,
        bwOpts.getBatchWriterConfig());
    BatchWriter dataBW = client.createBatchWriter(opts.dataTable, bwOpts.getBatchWriterConfig());
    FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility);
    for (String dir : opts.directories) {
      recurse(new File(dir), opts.visibility, dirBW, indexBW, fdi, dataBW);

      // fill in parent directory info
      int slashIndex = -1;
      while ((slashIndex = dir.lastIndexOf("/")) > 0) {
        dir = dir.substring(0, slashIndex);
        ingest(new File(dir), opts.visibility, dirBW, indexBW, fdi, dataBW);
      }
    }
    ingest(new File("/"), opts.visibility, dirBW, indexBW, fdi, dataBW);

    dirBW.close();
    indexBW.close();
    dataBW.close();
  }
}
 

public static void main(String[] args) throws Exception {
  IndexOpts opts = new IndexOpts();
  opts.parseArgs(Index.class.getName(), args);

  String splitRegex = "\\W+";

  try (AccumuloClient client = Accumulo.newClient().from(opts.getClientPropsPath()).build();
      BatchWriter bw = client.createBatchWriter(opts.tableName)) {
    for (String filename : opts.files) {
      index(opts.partitions, new File(filename), splitRegex, bw);
    }
  }
}
 

public static void main(String[] args) throws Exception {
  Opts opts = new Opts();
  opts.parseArgs(Reverse.class.getName(), args);

  try (AccumuloClient client = Accumulo.newClient().from(opts.getClientPropsPath()).build();
      Scanner scanner = client.createScanner(opts.shardTable, Authorizations.EMPTY);
      BatchWriter bw = client.createBatchWriter(opts.doc2TermTable)) {
    for (Entry<Key,Value> entry : scanner) {
      Key key = entry.getKey();
      Mutation m = new Mutation(key.getColumnQualifier());
      m.put(key.getColumnFamily(), new Text(), new Value(new byte[0]));
      bw.addMutation(m);
    }
  }
}