Java Code Examples for org.apache.accumulo.core.client.Scanner#fetchColumnFamily()

@Test
public void testFirstNEntriesReturned_multipleRowsLotsOfKeys_FetchRowColumn() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException {

    persistTestMutations(50, 500);

    Scanner scanner = buildScanner(10);
    scanner.setRange(new Range("1"));
    scanner.fetchColumnFamily(new Text("1"));

    assertEquals(1, Iterables.size(scanner));


    for (Map.Entry<Key, Value> entry : scanner) {
        Iterable<Map.Entry<Key, Value>> items = decodeRow(entry.getKey(), entry.getValue());
        assertEquals(1, Iterables.size(items));
    }
}
 

/**
 * Add one or more column families or column family and qualifier to a scanner.
 * 
 * @param scanner
 *            to add columns to.
 * @return The scanner with columns
 */
private Scanner addColumnsToScanner(Scanner scanner) {
    if ((null != columns) && (!columns.equals(("")))) {
        String[] cols = columns.split(",");
        for (String colStr : cols) {
            String[] parts = colStr.split(":");
            if (parts.length == 1) {
                scanner.fetchColumnFamily(new Text(parts[0]));
            } else if (parts.length == 2) {
                scanner.fetchColumn(new Text(parts[0]), new Text(parts[1]));
            }
        }
    }
    
    return (scanner);
}
 

@SuppressWarnings({"rawtypes", "unchecked"})
public Iterable<Index<? extends Element>> getIndices() {
  List<Index<? extends Element>> indexes = new ArrayList<Index<? extends Element>>();

  IndexedItemsListParser parser = new IndexedItemsListParser();

  Scanner scan = null;
  try {
    scan = getScanner();
    scan.fetchColumnFamily(new Text(IndexMetadataEntryType.__INDEX_NAME__.name()));

    for (IndexedItem item : parser.parse(scan)) {
      indexes.add(new AccumuloIndex(globals,
          item.getKey(), item.getElementClass()));
    }

    return indexes;

  } finally {
    if (scan != null) {
      scan.close();
    }
  }
}
 

public <T extends Element> Set<String> getIndexedKeys(Class<T> elementClass) {
  if (elementClass == null) {
    throw ExceptionFactory.classForElementCannotBeNull();
  }

  IndexedItemsListParser parser = new IndexedItemsListParser(elementClass);

  Scanner scan = null;
  try {
    scan = getScanner();
    scan.fetchColumnFamily(new Text(IndexMetadataEntryType.__INDEX_KEY__.name()));

    Set<String> keys = new HashSet<String>();
    for (IndexedItem item : parser.parse(scan)) {
      keys.add(item.getKey());
    }

    return keys;

  } finally {
    if (scan != null) {
      scan.close();
    }
  }
}
 

/**
 * Get elements with the key/value pair.
 * @param key
 * @param value
 * @return
 */
@SuppressWarnings("unchecked")
public <T extends Element> CloseableIterable<T> readElementsFromIndex(String key, Object value) {
  Scanner scan = getScanner();
  byte[] id = AccumuloByteSerializer.serialize(value);
  scan.setRange(Range.exact(new Text(id)));
  scan.fetchColumnFamily(new Text(key));

  final ElementIndexParser<? extends AccumuloElement> parser =
      Vertex.class.equals(elementType) ? new VertexIndexParser(globals) :
        new EdgeIndexParser(globals);

      return new ScannerIterable<T>(scan) {
        @Override
        public T next(PeekingIterator<Entry<Key,Value>> iterator) {
          return (T) parser.parse(Arrays.asList(iterator.next()));
        }      
      };
}
 

/**
 * Scan accumulo for the results that are stored in a PCJ table. The
 * multimap stores a set of deserialized binding sets that were in the PCJ
 * table for every variable order that is found in the PCJ metadata.
 */
private static Multimap<String, BindingSet> loadPcjResults(final Connector accumuloConn, final String pcjTableName) throws PcjException, TableNotFoundException, BindingSetConversionException {
    final Multimap<String, BindingSet> fetchedResults = HashMultimap.create();

    // Get the variable orders the data was written to.
    final PcjTables pcjs = new PcjTables();
    final PcjMetadata pcjMetadata = pcjs.getPcjMetadata(accumuloConn, pcjTableName);

    // Scan Accumulo for the stored results.
    for(final VariableOrder varOrder : pcjMetadata.getVarOrders()) {
        final Scanner scanner = accumuloConn.createScanner(pcjTableName, new Authorizations());
        scanner.fetchColumnFamily( new Text(varOrder.toString()) );

        for(final Entry<Key, Value> entry : scanner) {
            final byte[] serializedResult = entry.getKey().getRow().getBytes();
            final BindingSet result = converter.convert(serializedResult, varOrder);
            fetchedResults.put(varOrder.toString(), result);
        }
    }

    return fetchedResults;
}
 

/**
 * Read the given properties for the given element.
 * If propertyKeys is null, read all properties.
 * If the element has no properties, return an empty Map.
 * If the element does not exist, return null.
 * @param id
 * @param propertyKeys
 * @return
 */
public Map<String, Object> readProperties(Element element, String[] propertyKeys) {
  Scanner s = getScanner();
  s.setRange(Range.exact((String) element.getId()));

  // If propertyKeys is null, we read everything.
  // Otherwise, limit to the given attributes.
  if (propertyKeys != null) {
    s.fetchColumnFamily(new Text(Constants.LABEL));

    for (String key : propertyKeys) {
      s.fetchColumnFamily(new Text(key));
    }
  }

  Map<String, Object> props = new PropertyParser().parse(s);
  s.close();

  return props;
}
 

/**
 * Read the given property from the backing table
 * for the given element id.
 * @param id
 * @param key
 * @return
 */
public <V> V readProperty(Element element, String key) {
  Scanner s = getScanner();

  s.setRange(new Range(element.getId().toString()));

  Text colf = StringFactory.LABEL.equals(key)
      ? new Text(Constants.LABEL) : new Text(key);
  s.fetchColumnFamily(colf);

  V value = null;

  Iterator<Entry<Key, Value>> iter = s.iterator();
  if (iter.hasNext()) {
    value = AccumuloByteSerializer.deserialize(iter.next().getValue().get());
  }
  s.close();

  return value;
}
 

protected ScannerBase metricScanner(AccumuloFeatureConfig xform, Date start, Date end, String group, String type, String name, TimeUnit timeUnit, Auths auths) {
    checkNotNull(xform);

    try {

        group = defaultString(group);
        timeUnit = (timeUnit == null ? TimeUnit.MINUTES : timeUnit);

        Scanner scanner = connector.createScanner(tableName + REVERSE_SUFFIX, auths.getAuths());
        scanner.setRange(buildRange(type, start, end, timeUnit));

        if (group != null && name != null) {
            scanner.fetchColumn(new Text(combine(timeUnit.toString(), xform.featureName())), new Text(combine(group, name)));
        } else {
            scanner.fetchColumnFamily(new Text(combine(timeUnit.toString(), xform.featureName())));

            String cqRegex = null;
            if (group != null) {
                cqRegex = combine(group, "(.*)");
            } else if (name != null)
                cqRegex = combine("(.*)", name);
            if (cqRegex != null) {
                IteratorSetting regexIterator = new IteratorSetting(Constants.DEFAULT_ITERATOR_PRIORITY - 1, "regex", RegExFilter.class);
                scanner.addScanIterator(regexIterator);
            }
        }

        return scanner;
    } catch (TableNotFoundException e) {
        throw new RuntimeException(e);
    }
}
 

public void loadEndpointsAndLabel(AccumuloEdge edge) {
  Scanner s = getScanner();

  try {
    s.setRange(new Range(edge.getId().toString()));
    s.fetchColumnFamily(new Text(Constants.LABEL));
    Iterator<Entry<Key,Value>> iter = s.iterator();
    if (!iter.hasNext()) {
      dump();
      throw new AccumuloGraphException("Unable to find edge row: "+edge);
    }

    Entry<Key, Value> entry = iter.next();

    String cq = entry.getKey().getColumnQualifier().toString();
    String[] ids = cq.split(Constants.ID_DELIM);

    String label = AccumuloByteSerializer.deserialize(entry.getValue().get());

    edge.setVertices(new AccumuloVertex(globals, ids[0]),
        new AccumuloVertex(globals, ids[1]));
    edge.setLabel(label);

  } finally {
    s.close();
  }
}
 

/**
 * Checks to see if the provided term appears in other documents.
 * @param term the term to search for.
 * @param currentDocId the current document ID that the search term exists in.
 * @return {@code true} if the term was found in other documents. {@code false} otherwise.
 */
private boolean doesTermExistInOtherDocs(final String term, final int currentDocId, final Text docIdText) {
    try {
        final String freeTextDocTableName = getFreeTextDocTablename(conf);
        final Scanner scanner = getScanner(freeTextDocTableName);

        final String t = StringUtils.removeEnd(term, "*").toLowerCase();
        final Text queryTerm = ColumnPrefixes.getTermColFam(t);

        // perform query and read results
        scanner.fetchColumnFamily(queryTerm);

        for (final Entry<Key, Value> entry : scanner) {
            final Key key = entry.getKey();
            final Text row = key.getRow();
            final int rowId = Integer.parseInt(row.toString());
            // We only want to check other documents from the one we're deleting
            if (rowId != currentDocId) {
                final Text columnFamily = key.getColumnFamily();
                final String columnFamilyValue = columnFamily.toString();
                // Check that the value has the term prefix
                if (columnFamilyValue.startsWith(ColumnPrefixes.TERM_CF_PREFIX.toString())) {
                    final Text text = ColumnPrefixes.removePrefix(columnFamily);
                    final String value = text.toString();
                    if (value.equals(term)) {
                        return true;
                    }
                }
            }
        }
    } catch (final IOException e) {
        logger.error("Error searching for the existance of the term in other documents", e);
    }
    return false;
}
 

@Override
public List<InputSplit> getSplits(JobContext jobContext) throws IOException {
    //read the params from AccumuloInputFormat
    Configuration conf = jobContext.getConfiguration();
    Instance instance = MRUtils.AccumuloProps.getInstance(jobContext);
    String user = MRUtils.AccumuloProps.getUsername(jobContext);
    AuthenticationToken password = MRUtils.AccumuloProps.getPassword(jobContext);
    String table = MRUtils.AccumuloProps.getTablename(jobContext);
    ArgumentChecker.notNull(instance);
    ArgumentChecker.notNull(table);

    //find the files necessary
    try {
        Connector connector = instance.getConnector(user, password);
        TableOperations tos = connector.tableOperations();
        String tableId = tos.tableIdMap().get(table);
        Scanner scanner = connector.createScanner("accumulo.metadata", Authorizations.EMPTY); //TODO: auths?
        scanner.setRange(new Range(new Text(tableId + "\u0000"), new Text(tableId + "\uFFFD")));
        scanner.fetchColumnFamily(new Text("file"));
        List<String> files = new ArrayList<String>();
        List<InputSplit> fileSplits = new ArrayList<InputSplit>();
        for (Map.Entry<Key, Value> entry : scanner) {
            String file = entry.getKey().getColumnQualifier().toString();
            Path path = new Path(file);
            FileSystem fs = path.getFileSystem(conf);
            FileStatus fileStatus = fs.getFileStatus(path);
            long len = fileStatus.getLen();
            BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, len);
            files.add(file);
            fileSplits.add(new FileSplit(path, 0, len, fileBlockLocations[0].getHosts()));
        }
        System.out.println(files);
        return fileSplits;
    } catch (Exception e) {
        throw new IOException(e);
    }
}
 

public CloseableIterator<GeoWaveRow> getDataIndexResults(
    final byte[] startRow,
    final byte[] endRow,
    final short adapterId,
    final String... additionalAuthorizations) {
  final byte[] family = StringUtils.stringToBinary(ByteArrayUtils.shortToString(adapterId));

  // to have backwards compatibility before 1.8.0 we can assume BaseScanner is autocloseable
  final Scanner scanner;
  try {
    scanner = createScanner(DataIndexUtils.DATA_ID_INDEX.getName(), additionalAuthorizations);

    scanner.setRange(
        AccumuloUtils.byteArrayRangeToAccumuloRange(new ByteArrayRange(startRow, endRow)));
    scanner.fetchColumnFamily(new Text(family));
    return new CloseableIteratorWrapper(new Closeable() {
      @Override
      public void close() throws IOException {
        scanner.close();
      }
    },
        Streams.stream(scanner.iterator()).map(
            entry -> DataIndexUtils.deserializeDataIndexRow(
                entry.getKey().getRow().getBytes(),
                adapterId,
                entry.getValue().get(),
                false)).iterator());
  } catch (final TableNotFoundException e) {
    LOGGER.error("unable to find data index table", e);
  }
  return new CloseableIterator.Empty<>();
}
 

@Test
public void testUpdateState() throws Exception {
    // create the initial entry
    testPersistentCreate();
    
    PowerMock.resetAll();
    
    // Get ready to call updateState
    HashMap<String,String> trackingMap = new HashMap<>();
    expect(connectionFactory.getTrackingMap(EasyMock.anyObject())).andReturn(trackingMap);
    expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.ADMIN), EasyMock.eq(trackingMap))).andReturn(connection);
    connectionFactory.returnConnection(connection);
    expect(connectionFactory.getTrackingMap(EasyMock.anyObject())).andReturn(trackingMap);
    expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.ADMIN), EasyMock.eq(trackingMap))).andReturn(connection);
    connectionFactory.returnConnection(connection);
    replayAll();
    
    bean.updateState(mapReduceJobId, MapReduceState.FAILED);
    verifyAll();
    
    // Ensure that the new FAILED state made it into the table
    Key failedKey = new Key(id, sid, MapReduceStatePersisterBean.STATE + NULL + mapReduceJobId);
    Value failedValue = new Value(MapReduceState.FAILED.toString().getBytes());
    boolean found = false;
    Scanner s = connection.createScanner(TABLE_NAME, new Authorizations(auths));
    s.setRange(new Range(id));
    s.fetchColumnFamily(new Text(sid));
    
    for (Entry<Key,Value> entry : s) {
        if (entry.getKey().getColumnQualifier().toString().equals(MapReduceStatePersisterBean.STATE + NULL + mapReduceJobId)) {
            if (failedKey.equals(entry.getKey(), PartialKey.ROW_COLFAM_COLQUAL) && failedValue.equals(entry.getValue())) {
                found = true;
            }
        }
    }
    if (!found)
        fail("Updated state not found");
    
}
 

/**
 * Return true if the element with given id exists.
 * @param id
 * @return
 */
public boolean elementExists(String id) {
  Scanner scan = null;
  try {
    scan = getScanner();
    scan.setRange(Range.exact(id));
    scan.fetchColumnFamily(new Text(Constants.LABEL));
    return new PropertyParser().parse(scan) != null;

  } finally {
    if (scan != null) {
      scan.close();
    }
  }
}
 

@Test
@Category(UnitTest.class)
public void testGetTile() throws Exception
{

  ZooKeeperInstance zkinst = new ZooKeeperInstance(inst, zoo);
  PasswordToken pwTok = new PasswordToken(pw.getBytes());
  Connector conn = zkinst.getConnector(u, pwTok);
  Assert.assertNotNull(conn);

  PasswordToken token = new PasswordToken(pw.getBytes());

  Authorizations auths = new Authorizations(authsStr.split(","));
  long start = 0;
  long end = Long.MAX_VALUE;
  Key sKey = AccumuloUtils.toKey(start);
  Key eKey = AccumuloUtils.toKey(end);
  Range r = new Range(sKey, eKey);
  Scanner s = conn.createScanner("paris4", auths);
  s.fetchColumnFamily(new Text(Integer.toString(10)));
  s.setRange(r);

  Iterator<Entry<Key, Value>> it = s.iterator();
  while (it.hasNext())
  {
    Entry<Key, Value> ent = it.next();
    if (ent == null)
    {
      return;
    }
    System.out.println("current key   = " + AccumuloUtils.toLong(ent.getKey().getRow()));
    System.out.println("current value = " + ent.getValue().getSize());
  }

}
 

protected <T> Scanner getScanner(final RecordReaderParams params) {
  final GeoWaveRowRange range = params.getRowRange();
  final String tableName = params.getIndex().getName();
  Scanner scanner;
  try {
    scanner = createScanner(tableName, params.getAdditionalAuthorizations());
    if (range == null) {
      scanner.setRange(new Range());
    } else {
      scanner.setRange(
          AccumuloSplitsProvider.toAccumuloRange(
              range,
              params.getIndex().getIndexStrategy().getPartitionKeyLength()));
    }
    if ((params.getLimit() != null)
        && (params.getLimit() > 0)
        && (params.getLimit() < scanner.getBatchSize())) {
      // do allow the limit to be set to some enormous size.
      scanner.setBatchSize(Math.min(1024, params.getLimit()));
    }
    if (params.getMaxResolutionSubsamplingPerDimension() != null) {
      if (params.getMaxResolutionSubsamplingPerDimension().length != params.getIndex().getIndexStrategy().getOrderedDimensionDefinitions().length) {
        LOGGER.warn(
            "Unable to subsample for table '"
                + tableName
                + "'. Subsample dimensions = "
                + params.getMaxResolutionSubsamplingPerDimension().length
                + " when indexed dimensions = "
                + params.getIndex().getIndexStrategy().getOrderedDimensionDefinitions().length);
      } else {

        final int cardinalityToSubsample =
            (int) Math.round(
                IndexUtils.getDimensionalBitsUsed(
                    params.getIndex().getIndexStrategy(),
                    params.getMaxResolutionSubsamplingPerDimension())
                    + (8 * params.getIndex().getIndexStrategy().getPartitionKeyLength()));

        final IteratorSetting iteratorSettings =
            new IteratorSetting(
                FixedCardinalitySkippingIterator.CARDINALITY_SKIPPING_ITERATOR_PRIORITY,
                FixedCardinalitySkippingIterator.CARDINALITY_SKIPPING_ITERATOR_NAME,
                FixedCardinalitySkippingIterator.class);
        iteratorSettings.addOption(
            FixedCardinalitySkippingIterator.CARDINALITY_SKIP_INTERVAL,
            Integer.toString(cardinalityToSubsample));
        scanner.addScanIterator(iteratorSettings);
      }
    }
  } catch (final TableNotFoundException e) {
    LOGGER.warn("Unable to query table '" + tableName + "'.  Table does not exist.", e);
    return null;
  }
  if ((params.getAdapterIds() != null) && (params.getAdapterIds().length > 0)) {
    for (final Short adapterId : params.getAdapterIds()) {
      scanner.fetchColumnFamily(new Text(ByteArrayUtils.shortToString(adapterId)));
    }
  }
  return scanner;
}
 

@Override
public Set<Tag> getColumnFamilies(String metric, Map<String, String> requestedTags) throws TableNotFoundException {

    Scanner meta = null;
    try {
        Map<String, String> tags = (requestedTags == null) ? new LinkedHashMap<>() : requestedTags;
        LOG.trace("Looking for requested tags: {}", tags);
        meta = connector.createScanner(metaTable, Authorizations.EMPTY);
        Text start = new Text(Meta.VALUE_PREFIX + metric);
        Text end = new Text(Meta.VALUE_PREFIX + metric + "\\x0000");
        end.append(new byte[] { (byte) 0xff }, 0, 1);
        meta.setRange(new Range(start, end));
        // Only look for the meta entries that match our tags, if any
        boolean onlyFirstRow = false;
        Entry<String, String> first = null;
        // Set the columns on the meta scanner based on the first tag
        // in the set of tags passed in the query. If no tags are present
        // then we are only going to return the first tag name present in the
        // meta table.
        Iterator<Entry<String, String>> tagIter = tags.entrySet().iterator();
        if (tagIter.hasNext()) {
            first = tagIter.next();
            if (isTagValueRegex(first.getValue())) {
                meta.fetchColumnFamily(new Text(first.getKey()));
            } else {
                meta.fetchColumn(new Text(first.getKey()), new Text(first.getValue()));
            }
        } else {
            // grab all of the values found for the first tag for the metric
            onlyFirstRow = true;
        }
        final boolean ONLY_RETURN_FIRST_TAG = onlyFirstRow;
        Iterator<Entry<Key, Value>> iter = meta.iterator();
        Iterator<Pair<String, String>> knownKeyValues = new Iterator<Pair<String, String>>() {

            Text firstTag = null;
            Text tagName = null;
            Text tagValue = null;

            @Override
            public boolean hasNext() {
                if (iter.hasNext()) {
                    Entry<Key, Value> metaEntry = iter.next();
                    if (null == firstTag) {
                        firstTag = metaEntry.getKey().getColumnFamily();
                    }
                    tagName = metaEntry.getKey().getColumnFamily();
                    tagValue = metaEntry.getKey().getColumnQualifier();
                    LOG.trace("Found tag entry {}={}", tagName, tagValue);

                    if (ONLY_RETURN_FIRST_TAG && !tagName.equals(firstTag)) {
                        return false;
                    }
                    return true;
                }
                return false;
            }

            @Override
            public Pair<String, String> next() {
                LOG.trace("Returning tag {}={}", tagName, tagValue);
                return new Pair<>(tagName.toString(), tagValue.toString());
            }
        };
        // Expand the list of tags in the meta table for this metric that
        // matches
        // the pattern of the first tag in the query. The resulting set of tags
        // will be used to fetch specific columns from the metric table.
        return expandTagValues(first, knownKeyValues);
    } finally {
        if (meta != null) {
            meta.close();
        }
    }
}
 

public CloseableIterable<Edge> getEdges(Vertex vertex, Direction direction,
    String... labels) {
  Scanner scan = getScanner();
  scan.setRange(new Range(vertex.getId().toString()));
  if (direction.equals(Direction.IN)) {
    scan.fetchColumnFamily(new Text(Constants.IN_EDGE));
  } else if (direction.equals(Direction.OUT)) {
    scan.fetchColumnFamily(new Text(Constants.OUT_EDGE));
  } else {
    scan.fetchColumnFamily(new Text(Constants.IN_EDGE));
    scan.fetchColumnFamily(new Text(Constants.OUT_EDGE));
  }

  if (labels.length > 0) {
    applyEdgeLabelValueFilter(scan, labels);
  }

  return new ScannerIterable<Edge>(scan) {

    @Override
    public Edge next(PeekingIterator<Entry<Key,Value>> iterator) {
      // TODO better use of information readily available...
      // TODO could also check local cache before creating a new
      // instance?

      Entry<Key,Value> kv = iterator.next();

      String[] parts = kv.getKey().getColumnQualifier().toString().split(Constants.ID_DELIM);
      String label = (new String(kv.getValue().get())).split(Constants.ID_DELIM)[1];

      AccumuloEdge edge;
      if (kv.getKey().getColumnFamily().toString().equalsIgnoreCase(Constants.IN_EDGE)) {
        edge = new AccumuloEdge(globals, parts[1],
            new AccumuloVertex(globals, kv.getKey().getRow().toString()),
            new AccumuloVertex(globals, parts[0]), label);
      } else {
        edge = new AccumuloEdge(globals, parts[1],
            new AccumuloVertex(globals, parts[0]),
            new AccumuloVertex(globals, kv.getKey().getRow().toString()), label);
      }
      globals.getCaches().cache(edge, Edge.class);

      return edge;
    }
  };
}
 

/**
 * Get a list of timestamps that represents all of the Prospect runs that have
 * ever been performed.
 *
 * @param auths - The authorizations used to scan the table for prospects.
 * @return A list of timestamps representing each Prospect run that was found.
 * @throws TableNotFoundException The table name that was provided when this
 *   class was constructed does not match a table that the connector has access to.
 */
public Iterator<Long> getProspects(String[] auths) throws TableNotFoundException {
    final Scanner scanner = connector.createScanner(tableName, new Authorizations(auths));
    scanner.setRange(Range.exact(METADATA));
    scanner.fetchColumnFamily(new Text(PROSPECT_TIME));

    return new ProspectTimestampIterator( scanner.iterator() );
}