Java Code Examples for org.apache.accumulo.core.data.Mutation#getUpdates()

@Override
public void write(Text key, Mutation value) throws IOException, InterruptedException {
    try {
        for (ColumnUpdate update : value.getUpdates()) {
            log.debug("Table: "
                            + key
                            + ", Key: "
                            + new Key(value.getRow(), update.getColumnFamily(), update.getColumnQualifier(), update.getColumnVisibility(), update
                                            .getTimestamp()));
        }
        if (writerMap.get(key) == null) {
            log.error("key had null value: " + key);
        }
        writerMap.get(key).addMutation(value);
    } catch (MutationsRejectedException e) {
        throw new IOException("Error adding mutation", e);
    }
}
 

@Override
public List<Short> check(Environment env, Mutation mutation) {
  Set<Short> violations = null;

  if (!isAlphaNum(mutation.getRow()))
    violations = addViolation(violations, NON_ALPHA_NUM_ROW);

  Collection<ColumnUpdate> updates = mutation.getUpdates();
  for (ColumnUpdate columnUpdate : updates) {
    if (!isAlphaNum(columnUpdate.getColumnFamily()))
      violations = addViolation(violations, NON_ALPHA_NUM_COLF);

    if (!isAlphaNum(columnUpdate.getColumnQualifier()))
      violations = addViolation(violations, NON_ALPHA_NUM_COLQ);
  }

  return null == violations ? null : new ArrayList<>(violations);
}
 

synchronized void addMutation(Mutation m) {
  long now = System.currentTimeMillis();
  mutationCount++;
  for (ColumnUpdate u : m.getUpdates()) {
    Key key = new Key(m.getRow(), 0, m.getRow().length, u.getColumnFamily(), 0, u.getColumnFamily().length, u.getColumnQualifier(), 0,
        u.getColumnQualifier().length, u.getColumnVisibility(), 0, u.getColumnVisibility().length, u.getTimestamp());
    if (u.isDeleted())
      key.setDeleted(true);
    if (!u.hasTimestamp())
      if (timeType.equals(TimeType.LOGICAL))
        key.setTimestamp(mutationCount);
      else
        key.setTimestamp(now);
    
    table.put(new MockMemKey(key, mutationCount), new Value(u.getValue()));
  }
}
 

/**
 * Turn a mutation's column update into a key
 *
 * @param m
 *            the Mutation from which KeyValue pairs should be extracted
 * @return a List of KeyValue pairs representing the contents of {@code m}
 */
public static List<KeyValue> getKeyValues(Mutation m) {
    List<KeyValue> values = new ArrayList<>();
    for (ColumnUpdate update : m.getUpdates()) {
        values.add(new KeyValue(new Key(m.getRow(), update.getColumnFamily(), update.getColumnQualifier(), update.getColumnVisibility(), (update
                        .hasTimestamp() ? update.getTimestamp() : -1), update.isDeleted()), update.getValue()));
    }
    return values;
}
 

@Test
public void testWriteMultipleColumnsWithNonExpandedMap()
        throws IOException, ParseException {
    AccumuloStorage storage = new AccumuloStorage("col1,col2");

    Map<String, Object> map = Maps.newHashMap();

    map.put("mapcol1", "mapval1");
    map.put("mapcol2", "mapval2");
    map.put("mapcol3", "mapval3");
    map.put("mapcol4", "mapval4");

    Tuple t = TupleFactory.getInstance().newTuple(3);
    t.set(0, "row");
    t.set(1, "value1");
    t.set(2, map);

    Collection<Mutation> mutations = storage.getMutations(t);

    Assert.assertEquals(1, mutations.size());

    Mutation m = mutations.iterator().next();

    Assert.assertTrue("Rows not equal",
            Arrays.equals(m.getRow(), ((String) t.get(0)).getBytes()));

    List<ColumnUpdate> colUpdates = m.getUpdates();
    Assert.assertEquals(2, colUpdates.size());

    ColumnUpdate update = colUpdates.get(0);
    Assert.assertEquals("col1", new String(update.getColumnFamily()));
    Assert.assertEquals("", new String(update.getColumnQualifier()));
    Assert.assertEquals("value1", new String(update.getValue()));

    update = colUpdates.get(1);
    Assert.assertEquals("col2", new String(update.getColumnFamily()));
    Assert.assertEquals("", new String(update.getColumnQualifier()));
    Assert.assertArrayEquals(storage.objToBytes(map, DataType.MAP),
            update.getValue());
}
 

@Test
public void testWriteIgnoredExtraMap() throws IOException, ParseException {
    AccumuloStorage storage = new AccumuloStorage("col1");

    Map<String, Object> map = Maps.newHashMap();

    map.put("mapcol1", "mapval1");
    map.put("mapcol2", "mapval2");
    map.put("mapcol3", "mapval3");
    map.put("mapcol4", "mapval4");

    Tuple t = TupleFactory.getInstance().newTuple(3);
    t.set(0, "row");
    t.set(1, "value1");
    t.set(2, map);

    Collection<Mutation> mutations = storage.getMutations(t);

    Assert.assertEquals(1, mutations.size());

    Mutation m = mutations.iterator().next();

    Assert.assertTrue("Rows not equal",
            Arrays.equals(m.getRow(), ((String) t.get(0)).getBytes()));

    List<ColumnUpdate> colUpdates = m.getUpdates();
    Assert.assertEquals(1, colUpdates.size());

    ColumnUpdate update = colUpdates.get(0);
    Assert.assertEquals("col1", new String(update.getColumnFamily()));
    Assert.assertEquals("", new String(update.getColumnQualifier()));
    Assert.assertEquals("value1", new String(update.getValue()));
}
 

@Test
public void testWriteIgnoredExtraColumns() throws IOException,
        ParseException {
    AccumuloStorage storage = new AccumuloStorage("col");

    Tuple t = TupleFactory.getInstance().newTuple(3);
    t.set(0, "row");
    t.set(1, "value1");
    t.set(2, "value2");

    Collection<Mutation> mutations = storage.getMutations(t);

    Assert.assertEquals(1, mutations.size());

    Mutation m = mutations.iterator().next();

    Assert.assertTrue("Rows not equal",
            Arrays.equals(m.getRow(), ((String) t.get(0)).getBytes()));

    List<ColumnUpdate> colUpdates = m.getUpdates();
    Assert.assertEquals(1, colUpdates.size());

    ColumnUpdate colUpdate = colUpdates.get(0);
    Assert.assertTrue("CF not equal",
            Arrays.equals(colUpdate.getColumnFamily(), "col".getBytes()));
    Assert.assertTrue("CQ not equal",
            Arrays.equals(colUpdate.getColumnQualifier(), new byte[0]));
    Assert.assertTrue("Values not equal",
            Arrays.equals(colUpdate.getValue(), "value1".getBytes()));
}
 

@Test
public void testWrite2TupleWithColumnQual() throws IOException,
        ParseException {
    AccumuloStorage storage = new AccumuloStorage("col:qual");

    Tuple t = TupleFactory.getInstance().newTuple(2);
    t.set(0, "row");
    t.set(1, "value");

    Collection<Mutation> mutations = storage.getMutations(t);

    Assert.assertEquals(1, mutations.size());

    Mutation m = mutations.iterator().next();

    Assert.assertTrue("Rows not equal",
            Arrays.equals(m.getRow(), ((String) t.get(0)).getBytes()));

    List<ColumnUpdate> colUpdates = m.getUpdates();
    Assert.assertEquals(1, colUpdates.size());

    ColumnUpdate colUpdate = colUpdates.get(0);
    Assert.assertTrue("CF not equal",
            Arrays.equals(colUpdate.getColumnFamily(), "col".getBytes()));
    Assert.assertTrue("CQ not equal", Arrays.equals(
            colUpdate.getColumnQualifier(), "qual".getBytes()));
    Assert.assertTrue("Values not equal",
            Arrays.equals(colUpdate.getValue(), "value".getBytes()));
}
 

@Test
public void testWrite2TupleWithColumn() throws IOException, ParseException {
    AccumuloStorage storage = new AccumuloStorage("col");

    Tuple t = TupleFactory.getInstance().newTuple(2);
    t.set(0, "row");
    t.set(1, "value");

    Collection<Mutation> mutations = storage.getMutations(t);

    Assert.assertEquals(1, mutations.size());

    Mutation m = mutations.iterator().next();

    Assert.assertTrue("Rows not equal",
            Arrays.equals(m.getRow(), ((String) t.get(0)).getBytes()));

    List<ColumnUpdate> colUpdates = m.getUpdates();
    Assert.assertEquals(1, colUpdates.size());

    ColumnUpdate colUpdate = colUpdates.get(0);
    Assert.assertTrue("CF not equal",
            Arrays.equals(colUpdate.getColumnFamily(), "col".getBytes()));
    Assert.assertTrue("CQ not equal",
            Arrays.equals(colUpdate.getColumnQualifier(), new byte[0]));
    Assert.assertTrue("Values not equal",
            Arrays.equals(colUpdate.getValue(), "value".getBytes()));
}
 

/**
 * Writes a mutation to the specified table.  If the mutation is meant to delete then the mutation will
 * be transformed to a delete mutation.
 * @param table the table to write to.
 * @param mutation the {@link mutation}.
 * @param context the {@link Context}.
 * @param isDelete {@code true} if the mutation should be a delete mutation.  {@code false} otherwise.
 * @throws IOException
 * @throws InterruptedException
 */
private static void writeMutation(final Text table, final Mutation mutation, final Context context, final boolean isDelete) throws IOException, InterruptedException {
    if (isDelete) {
        final List<ColumnUpdate> updates = mutation.getUpdates();
        final ColumnUpdate columnUpdate = updates.get(0);
        final ColumnVisibility cv = columnUpdate.getColumnVisibility() != null ? new ColumnVisibility(columnUpdate.getColumnVisibility()) : null;
        final Mutation deleteMutation = new Mutation(new Text(mutation.getRow()));
        deleteMutation.putDelete(columnUpdate.getColumnFamily(), columnUpdate.getColumnQualifier(), cv, columnUpdate.getTimestamp());
        context.write(table, deleteMutation);
    } else {
        context.write(table, mutation);
    }
}
 

@Override
public List<Short> check(Environment env, Mutation mutation) {
  Collection<ColumnUpdate> updates = mutation.getUpdates();

  for (ColumnUpdate columnUpdate : updates) {
    if (!isNumeric(columnUpdate.getValue()))
      return VIOLATION_LIST;
  }

  return null;
}
 

void put(Map<Key, Value> testData, Metric m) {
    Mutation mutation = MetricAdapter.toMutation(m);
    for (ColumnUpdate cu : mutation.getUpdates()) {
        Key key = new Key(mutation.getRow(), cu.getColumnFamily(), cu.getColumnQualifier(),
                cu.getColumnVisibility(), cu.getTimestamp());
        testData.put(key, new Value(cu.getValue()));
    }
}
 

private void createTestData1() {
    List<Tag> tags = Collections.singletonList(new Tag("host", "host1"));

    for (long i = 0; i < 1000; i += 100) {
        Metric m = new Metric("sys.loadAvg", i, .2, tags);
        Mutation mutation = MetricAdapter.toMutation(m);
        for (ColumnUpdate cu : mutation.getUpdates()) {
            Key key = new Key(mutation.getRow(), cu.getColumnFamily(), cu.getColumnQualifier(),
                    cu.getColumnVisibility(), cu.getTimestamp());
            System.out.println(key.toString());
            testData1.put(key, new Value(cu.getValue()));
        }
    }
}
 

private int printMutation(Text table, Mutation m) {
    if (log.isTraceEnabled()) {
        log.trace(String.format("Table %s row key: %s", table, hexDump(m.getRow())));
        for (ColumnUpdate cu : m.getUpdates()) {
            log.trace(String.format("Table %s column: %s:%s", table, hexDump(cu.getColumnFamily()), hexDump(cu.getColumnQualifier())));
            log.trace(String.format("Table %s security: %s", table, new ColumnVisibility(cu.getColumnVisibility()).toString()));
            log.trace(String.format("Table %s value: %s", table, hexDump(cu.getValue())));
        }
    }
    return m.getUpdates().size();
}
 

@Test
public void testWriteMapWithColFamColQualPrefix() throws IOException,
        ParseException {
    AccumuloStorage storage = new AccumuloStorage("col:qual_*");

    Map<String, Object> map = Maps.newHashMap();

    map.put("mapcol1", "mapval1");
    map.put("mapcol2", "mapval2");
    map.put("mapcol3", "mapval3");
    map.put("mapcol4", "mapval4");

    Tuple t = TupleFactory.getInstance().newTuple(2);
    t.set(0, "row");
    t.set(1, map);

    Collection<Mutation> mutations = storage.getMutations(t);

    Assert.assertEquals(1, mutations.size());

    Mutation m = mutations.iterator().next();

    Assert.assertTrue("Rows not equal",
            Arrays.equals(m.getRow(), ((String) t.get(0)).getBytes()));

    List<ColumnUpdate> colUpdates = m.getUpdates();
    Assert.assertEquals(4, colUpdates.size());

    Map<Entry<String, String>, String> expectations = Maps.newHashMap();
    expectations.put(Maps.immutableEntry("col", "qual_mapcol1"), "mapval1");
    expectations.put(Maps.immutableEntry("col", "qual_mapcol2"), "mapval2");
    expectations.put(Maps.immutableEntry("col", "qual_mapcol3"), "mapval3");
    expectations.put(Maps.immutableEntry("col", "qual_mapcol4"), "mapval4");

    for (ColumnUpdate update : colUpdates) {
        Entry<String, String> key = Maps.immutableEntry(
                new String(update.getColumnFamily()),
                new String(update.getColumnQualifier()));
        String value = new String(update.getValue());
        Assert.assertTrue(expectations.containsKey(key));

        String actual = expectations.remove(key);
        Assert.assertEquals(value, actual);
    }

    Assert.assertTrue("Did not find all expectations",
            expectations.isEmpty());
}
 

@Test
public void testWrite2TupleWithMixedColumns() throws IOException,
        ParseException {
    AccumuloStorage storage = new AccumuloStorage(
            "col1,col1:qual,col2:qual,col2");

    Tuple t = TupleFactory.getInstance().newTuple(5);
    t.set(0, "row");
    t.set(1, "value1");
    t.set(2, "value2");
    t.set(3, "value3");
    t.set(4, "value4");

    Collection<Mutation> mutations = storage.getMutations(t);

    Assert.assertEquals(1, mutations.size());

    Mutation m = mutations.iterator().next();

    Assert.assertTrue("Rows not equal",
            Arrays.equals(m.getRow(), ((String) t.get(0)).getBytes()));

    List<ColumnUpdate> colUpdates = m.getUpdates();
    Assert.assertEquals(4, colUpdates.size());

    ColumnUpdate colUpdate = colUpdates.get(0);
    Assert.assertTrue("CF not equal",
            Arrays.equals(colUpdate.getColumnFamily(), "col1".getBytes()));
    Assert.assertTrue("CQ not equal",
            Arrays.equals(colUpdate.getColumnQualifier(), new byte[0]));
    Assert.assertTrue("Values not equal",
            Arrays.equals(colUpdate.getValue(), "value1".getBytes()));

    colUpdate = colUpdates.get(1);
    Assert.assertTrue("CF not equal",
            Arrays.equals(colUpdate.getColumnFamily(), "col1".getBytes()));
    Assert.assertTrue("CQ not equal", Arrays.equals(
            colUpdate.getColumnQualifier(), "qual".getBytes()));
    Assert.assertTrue("Values not equal",
            Arrays.equals(colUpdate.getValue(), "value2".getBytes()));

    colUpdate = colUpdates.get(2);
    Assert.assertTrue("CF not equal",
            Arrays.equals(colUpdate.getColumnFamily(), "col2".getBytes()));
    Assert.assertTrue("CQ not equal", Arrays.equals(
            colUpdate.getColumnQualifier(), "qual".getBytes()));
    Assert.assertTrue("Values not equal",
            Arrays.equals(colUpdate.getValue(), "value3".getBytes()));

    colUpdate = colUpdates.get(3);
    Assert.assertTrue("CF not equal",
            Arrays.equals(colUpdate.getColumnFamily(), "col2".getBytes()));
    Assert.assertTrue("CQ not equal",
            Arrays.equals(colUpdate.getColumnQualifier(), new byte[0]));
    Assert.assertTrue("Values not equal",
            Arrays.equals(colUpdate.getValue(), "value4".getBytes()));
}
 

/**
 * Index the given mutation, adding mutations to the index and metrics table
 * <p>
 * Like typical use of a BatchWriter, this method does not flush mutations to the underlying index table.
 * For higher throughput the modifications to the metrics table are tracked in memory and added to the metrics table when the indexer is flushed or closed.
 *
 * @param mutation Mutation to index
 */
public void index(Mutation mutation)
{
    // Increment the cardinality for the number of rows in the table
    metrics.get(METRICS_TABLE_ROW_COUNT).incrementAndGet();

    // Set the first and last row values of the table based on existing row IDs
    if (firstRow == null || byteArrayComparator.compare(mutation.getRow(), firstRow) < 0) {
        firstRow = mutation.getRow();
    }

    if (lastRow == null || byteArrayComparator.compare(mutation.getRow(), lastRow) > 0) {
        lastRow = mutation.getRow();
    }

    // For each column update in this mutation
    for (ColumnUpdate columnUpdate : mutation.getUpdates()) {
        // Get the column qualifiers we want to index for this column family (if any)
        ByteBuffer family = wrap(columnUpdate.getColumnFamily());
        Collection<ByteBuffer> indexQualifiers = indexColumns.get(family);

        // If we have column qualifiers we want to index for this column family
        if (indexQualifiers != null) {
            // Check if we want to index this particular qualifier
            ByteBuffer qualifier = wrap(columnUpdate.getColumnQualifier());
            if (indexQualifiers.contains(qualifier)) {
                // If so, create a mutation using the following mapping:
                // Row ID = column value
                // Column Family = columnqualifier_columnfamily
                // Column Qualifier = row ID
                // Value = empty
                ByteBuffer indexFamily = getIndexColumnFamily(columnUpdate.getColumnFamily(), columnUpdate.getColumnQualifier());
                Type type = indexColumnTypes.get(family).get(qualifier);
                ColumnVisibility visibility = new ColumnVisibility(columnUpdate.getColumnVisibility());

                // If this is an array type, then index each individual element in the array
                if (Types.isArrayType(type)) {
                    Type elementType = Types.getElementType(type);
                    List<?> elements = serializer.decode(type, columnUpdate.getValue());
                    for (Object element : elements) {
                        addIndexMutation(wrap(serializer.encode(elementType, element)), indexFamily, visibility, mutation.getRow());
                    }
                }
                else {
                    addIndexMutation(wrap(columnUpdate.getValue()), indexFamily, visibility, mutation.getRow());
                }
            }
        }
    }
}
 

@Test
public void testWriteMultipleColumnsWithExpandedMap() throws IOException,
        ParseException {
    AccumuloStorage storage = new AccumuloStorage("col1,col2:");

    Map<String, Object> map = Maps.newHashMap();

    map.put("mapcol1", "mapval1");
    map.put("mapcol2", "mapval2");
    map.put("mapcol3", "mapval3");
    map.put("mapcol4", "mapval4");

    Tuple t = TupleFactory.getInstance().newTuple(3);
    t.set(0, "row");
    t.set(1, "value1");
    t.set(2, map);

    Collection<Mutation> mutations = storage.getMutations(t);

    Assert.assertEquals(1, mutations.size());

    Mutation m = mutations.iterator().next();

    Assert.assertTrue("Rows not equal",
            Arrays.equals(m.getRow(), ((String) t.get(0)).getBytes()));

    List<ColumnUpdate> colUpdates = m.getUpdates();
    Assert.assertEquals(5, colUpdates.size());

    ColumnUpdate update = colUpdates.get(0);
    Assert.assertEquals("col1", new String(update.getColumnFamily()));
    Assert.assertEquals("", new String(update.getColumnQualifier()));
    Assert.assertEquals("value1", new String(update.getValue()));

    Map<Entry<String, String>, String> expectations = Maps.newHashMap();
    expectations.put(Maps.immutableEntry("col2", "mapcol1"), "mapval1");
    expectations.put(Maps.immutableEntry("col2", "mapcol2"), "mapval2");
    expectations.put(Maps.immutableEntry("col2", "mapcol3"), "mapval3");
    expectations.put(Maps.immutableEntry("col2", "mapcol4"), "mapval4");

    for (int i = 1; i < 5; i++) {
        update = colUpdates.get(i);
        Entry<String, String> key = Maps.immutableEntry(
                new String(update.getColumnFamily()),
                new String(update.getColumnQualifier()));
        String value = new String(update.getValue());
        Assert.assertTrue("Did not find expected key: " + key,
                expectations.containsKey(key));

        String actual = expectations.remove(key);
        Assert.assertEquals(value, actual);
    }

    Assert.assertTrue("Did not find all expectations",
            expectations.isEmpty());
}
 

@Test
public void testWriteMapWithColFamWithColon() throws IOException,
        ParseException {
    AccumuloStorage storage = new AccumuloStorage("col:");

    Map<String, Object> map = Maps.newHashMap();

    map.put("mapcol1", "mapval1");
    map.put("mapcol2", "mapval2");
    map.put("mapcol3", "mapval3");
    map.put("mapcol4", "mapval4");

    Tuple t = TupleFactory.getInstance().newTuple(2);
    t.set(0, "row");
    t.set(1, map);

    Collection<Mutation> mutations = storage.getMutations(t);

    Assert.assertEquals(1, mutations.size());

    Mutation m = mutations.iterator().next();

    Assert.assertTrue("Rows not equal",
            Arrays.equals(m.getRow(), ((String) t.get(0)).getBytes()));

    List<ColumnUpdate> colUpdates = m.getUpdates();
    Assert.assertEquals(4, colUpdates.size());

    Map<Entry<String, String>, String> expectations = Maps.newHashMap();
    expectations.put(Maps.immutableEntry("col", "mapcol1"), "mapval1");
    expectations.put(Maps.immutableEntry("col", "mapcol2"), "mapval2");
    expectations.put(Maps.immutableEntry("col", "mapcol3"), "mapval3");
    expectations.put(Maps.immutableEntry("col", "mapcol4"), "mapval4");

    for (ColumnUpdate update : colUpdates) {
        Entry<String, String> key = Maps.immutableEntry(
                new String(update.getColumnFamily()),
                new String(update.getColumnQualifier()));
        String value = new String(update.getValue());
        Assert.assertTrue("Did not find expected key: " + key,
                expectations.containsKey(key));

        String actual = expectations.remove(key);
        Assert.assertEquals(value, actual);
    }

    Assert.assertTrue("Did not find all expectations",
            expectations.isEmpty());
}
 

@Test
public void testWriteMapWithColFamWithColonAsterisk() throws IOException,
        ParseException {
    AccumuloStorage storage = new AccumuloStorage("col:*");

    Map<String, Object> map = Maps.newHashMap();

    map.put("mapcol1", "mapval1");
    map.put("mapcol2", "mapval2");
    map.put("mapcol3", "mapval3");
    map.put("mapcol4", "mapval4");

    Tuple t = TupleFactory.getInstance().newTuple(2);
    t.set(0, "row");
    t.set(1, map);

    Collection<Mutation> mutations = storage.getMutations(t);

    Assert.assertEquals(1, mutations.size());

    Mutation m = mutations.iterator().next();

    Assert.assertTrue("Rows not equal",
            Arrays.equals(m.getRow(), ((String) t.get(0)).getBytes()));

    List<ColumnUpdate> colUpdates = m.getUpdates();
    Assert.assertEquals(4, colUpdates.size());

    Map<Entry<String, String>, String> expectations = Maps.newHashMap();
    expectations.put(Maps.immutableEntry("col", "mapcol1"), "mapval1");
    expectations.put(Maps.immutableEntry("col", "mapcol2"), "mapval2");
    expectations.put(Maps.immutableEntry("col", "mapcol3"), "mapval3");
    expectations.put(Maps.immutableEntry("col", "mapcol4"), "mapval4");

    for (ColumnUpdate update : colUpdates) {
        Entry<String, String> key = Maps.immutableEntry(
                new String(update.getColumnFamily()),
                new String(update.getColumnQualifier()));
        String value = new String(update.getValue());
        Assert.assertTrue("Did not find expected key: " + key,
                expectations.containsKey(key));

        String actual = expectations.remove(key);
        Assert.assertEquals(value, actual);
    }

    Assert.assertTrue("Did not find all expectations",
            expectations.isEmpty());
}