Java Code Examples for com.google.common.collect.TreeMultimap#put()

/**
 * Read each line of input, tokenize them into key/value pairs, perform a lookup/transformation of the keys via the
 * MetadataDictionary where applicable, and return the results as a map.
 * 
 * @param input the bytes to convert
 * @param output a ByteArrayOutputStream to write failed parse attempts to
 * @return a Map containing each line converted to a key/value pair and sorted alphabetically by key
 * @throws IOException If there is some I/O problem.
 */
public Map<String, Collection<String>> convertLinesToMap(final byte[] input, final ByteArrayOutputStream output) throws IOException {
    final TreeMultimap<String, String> kv = TreeMultimap.create();
    final LineTokenizer ltok = new LineTokenizer(input, this.charset);

    // Look at each line for a key value and run it through the dictionary
    while (ltok.hasMoreTokens()) {
        final String line = ltok.nextToken();
        final int pos = line.indexOf(SEP);
        if (pos == -1) {
            output.write(line.getBytes());
            output.write('\n');
            this.logger.debug("Found no key/value pair on line " + line);
        } else {
            final String key = line.substring(0, pos);
            final String value = line.substring(pos + 1);
            final String nkey = this.dict.map(this.servicePrefix + key);
            kv.put(nkey, value.trim());
            this.logger.debug("Mapped key " + key + " to " + nkey + ": " + value);
        }
    }
    return new TreeMap<>(kv.asMap());
}
 

@Test
public void test() throws Throwable {
    List<String> filterSet1 = Lists.newArrayList("a", "b", "c", "q", "r", "s");
    Itr<String> f1 = new Itr<>(filterSet1);
    List<String> filterSet2 = Lists.newArrayList("a", "b", "c", "d", "e", "f");
    Itr<String> f2 = new Itr<>(filterSet2);
    TreeMultimap<String,NestedIterator<String>> mmap = TreeMultimap.create(Util.keyComparator(), Util.hashComparator());
    mmap.put(f1.next(), f1);
    mmap.put(f2.next(), f2);
    
    assertTrue(NegationFilter.isFiltered("e", mmap, Util.keyTransformer()));
    assertEquals(2, mmap.keySet().size());
    assertNotEquals(null, mmap.get("e"));
    assertNotEquals(null, mmap.get("q"));
    assertEquals(2, mmap.values().size());
}
 

public static Collection<AddressSpecHash> getAddressesByAge(WalletDatabase db, NetworkParams params)
{
  TreeMultimap<Long, AddressSpecHash> age_map = TreeMultimap.create();
  for(AddressSpec spec : db.getAddressesList())
  {
    String addr = AddressUtil.getAddressString(spec, params);
    long tm = 0;
    if (db.getAddressCreateTimeMap().containsKey(addr))
    {
      tm = db.getAddressCreateTimeMap().get(addr);
    }
    age_map.put(tm, AddressUtil.getHashForSpec(spec));

  }

  return age_map.values();

}
 

/**
 * Creates a sorted mapping of values to iterators.
 *
 * @param subtree
 * @param sources
 * @return
 */
private static <T extends Comparable<T>> TreeMultimap<T,NestedIterator<T>> initSubtree(TreeMultimap<T,NestedIterator<T>> subtree,
                Iterable<NestedIterator<T>> sources, Transformer<T> transformer, Map<T,T> transforms, boolean anded) {
    for (NestedIterator<T> src : sources) {
        src.initialize();
        if (src.hasNext()) {
            T next = src.next();
            T transform = transformer.transform(next);
            if (transforms != null) {
                transforms.put(transform, next);
            }
            subtree.put(transform, src);
        } else if (anded) {
            // If a source has no valid records, it shouldn't throw an exception. It should just return no results.
            // For an And, once one source is exhausted, the entire tree is exhausted
            return Util.getEmpty();
        }
    }
    return subtree;
}
 

static TreeMultimap<String,IndexStream> nextAll(String key, Collection<IndexStream> streams) {
    TreeMultimap<String,IndexStream> newChildren = TreeMultimap.create(Ordering.natural(), Ordering.arbitrary());
    for (IndexStream itr : streams) {
        if (!isDay(key) && isDay(key(itr.peek()))) {
            newChildren.put(itr.peek().first(), itr);
        } else {
            itr.next();
            if (itr.hasNext()) {
                newChildren.put(itr.peek().first(), itr);
            } else {
                return TreeMultimap.create(Ordering.natural(), Ordering.arbitrary());
            }
        }
    }
    return newChildren;
}
 

private TreeMultimap<String, MatchCandidate> findCandidatesFor(Map<String, Field> fields) throws StageException {
  TreeMultimap<String, MatchCandidate> candidates = TreeMultimap.create();

  for (String outputField : outputFieldNames) {
    for (Map.Entry<String, Field> entry : fields.entrySet()) {
      String fieldPath = entry.getKey();
      Field field = entry.getValue();
      int score = FuzzyMatch.getRatio(outputField, fieldPath, true);
      if (score >= matchThreshold) {
        if (greaterScore(candidates.get(outputField), score) || allCandidates) {
          if (!allCandidates) {
            // If not storing all candidates we must clear any existing candidates for this key
            // since a Multimap won't replace multiple values for a key.
            candidates.get(outputField).clear();
          }
          candidates.put(outputField, new MatchCandidate(outputField, score, fieldPath, field));
        }
      }
    }
  }

  return candidates;
}
 

public static TreeMultimap<String, byte[]> tokenUpdate(byte[] key, Multimap<String, String> lookup)
		throws InvalidKeyException, InvalidAlgorithmParameterException, NoSuchAlgorithmException,
		NoSuchProviderException, NoSuchPaddingException, IOException {

	// A lexicographic sorted tree to hide order of insertion
	TreeMultimap<String, byte[]> tokenUp = TreeMultimap.create(Ordering.natural(), Ordering.usingToString());
	// Key generation
	SecureRandom random = new SecureRandom();
	random.setSeed(CryptoPrimitives.randomSeed(16));
	byte[] iv = new byte[16];

	for (String word : lookup.keySet()) {

		byte[] key1 = CryptoPrimitives.generateCmac(key, 1 + new String());

		byte[] key2 = CryptoPrimitives.generateCmac(key, 2 + word);

		for (String id : lookup.get(word)) {
			random.nextBytes(iv);
			int counter = 0;

			if (state.get(word) != null) {
				counter = state.get(word);
			}

			state.put(word, counter + 1);

			byte[] l = CryptoPrimitives.generateCmac(key2, "" + counter);

			byte[] value = CryptoPrimitives.encryptAES_CTR_String(key1, iv, id, sizeOfFileIdentifer);
			tokenUp.put(new String(l), value);
		}

	}
	return tokenUp;
}
 

private void calculateResults() {
    TreeMultimap<Integer, String> rankings = TreeMultimap.create();
    for (Player player : players.values()) {

        int score = 0;
        List<Answer> playerAnswers = new ArrayList<>();
        //cannot simply copy the values() of player.getAnswers(), because it is an unordered map (as it needs to be concurrent)
        for (Piece piece : pieces) {
            Answer answer = player.getAnswers().get(piece.getId());
            if (answer.getTempo() > -1) {
                int diff = Math.abs(answer.getTempo() - piece.getTempo());
                if (diff < 3) {
                    score += 15;
                } else {
                    score += 5 / Math.log10(diff);
                }
            }
            if (answer.getMainInstrument() == piece.getMainInstrument()) {
                score += 10;
            }
            if (answer.getMetreNumerator() == piece.getMetreNumerator() && answer.getMetreDenominator() == piece.getMetreDenominator()) {
                score += 10;
            }
            playerAnswers.add(answer);
        }
        results.getScores().put(player.getName(), score);
        rankings.put(score, player.getSession().getId());
    }
    // the ordered player ids
    results.setRanking(new ArrayList<>(rankings.values()));
    Collections.reverse(results.getRanking());
}
 

@Test
public void testNoAggregatorWithDeepCopy() throws IOException {
    TreeMultimap<Key,Value> map = TreeMultimap.create();
    
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithUid("abc.0").build().toByteArray()));
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithRemoveUid("abc.0").build().toByteArray()));
    
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abd"), new Value(createValueWithUid("abc.3").build().toByteArray()));
    
    SortedMultiMapIterator data = new SortedMultiMapIterator(map);
    
    PropogatingIterator iter = new PropogatingIterator();
    Map<String,String> options = Maps.newHashMap();
    
    IteratorEnvironment env = new MockIteratorEnvironment(false);
    
    iter.init(data, options, env);
    iter = iter.deepCopy(env);
    
    iter.seek(new Range(), Collections.emptyList(), false);
    
    Assert.assertTrue(iter.hasTop());
    
    Key topKey = iter.getTopKey();
    
    Assert.assertEquals(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), topKey);
    validateUids(iter.getTopValue(), "abc.0");
    iter.next();
    Assert.assertEquals(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), topKey);
    validateRemoval(iter.getTopValue(), "abc.0");
    iter.next();
    topKey = iter.getTopKey();
    Assert.assertEquals(newKey(SHARD, FIELD_TO_AGGREGATE, "abd"), topKey);
    
}
 

static TreeMultimap<String,IndexStream> pivot(TreeMultimap<String,IndexStream> children) {
    TreeMultimap<String,IndexStream> newChildren = TreeMultimap.create(Ordering.natural(), Ordering.arbitrary());
    final String max = children.keySet().last();
    newChildren.putAll(max, children.removeAll(max));
    for (IndexStream itr : children.values()) {
        // move this iterator forward until we get to a key past the max processed thus far
        String dayOrShard = null;
        while (itr.hasNext()) {
            
            dayOrShard = key(itr.peek());
            if (dayOrShard.compareTo(max) >= 0) {
                break;
            }
            if (isDay(dayOrShard) && max.startsWith(dayOrShard)) {
                // use the existing max instead of the day to add to the list
                dayOrShard = max;
                break;
            }
            itr.next();
        }
        // add the item into our map
        if (itr.hasNext()) {
            newChildren.put(dayOrShard, itr);
        } else {
            // nobody has anything past max, so no intersection
            return TreeMultimap.create(Ordering.natural(), Ordering.arbitrary());
        }
    }
    return newChildren;
}
 

private SortedMultiMapIterator createSourceWithTestData() {
    TreeMultimap<Key,Value> map = TreeMultimap.create();
    
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc", true), new Value(createValueWithUid("abc.0").build().toByteArray()));
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithUid("abc.1").build().toByteArray()));
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithUid("abc.2").build().toByteArray()));
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithUid("abc.3").build().toByteArray()));
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithUid("abc.4").build().toByteArray()));
    
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abd"), new Value(createValueWithUid("abc.3").build().toByteArray()));
    
    return new SortedMultiMapIterator(map);
}
 

private @Nullable Collection<IYModel> internalFetch(ITmfStateSystem ss, Map<String, Object> fetchParameters,
        @Nullable IProgressMonitor monitor) throws StateSystemDisposedException {
    SelectedCounterQueryFilter filter = createCounterQuery(fetchParameters);
    if (filter == null) {
        return null;
    }
    long stateSystemEndTime = ss.getCurrentEndTime();
    Collection<Long> times = extractRequestedTimes(ss, filter, stateSystemEndTime);

    Map<Long, Integer> entries = Maps.filterValues(getSelectedEntries(filter), q -> ss.getSubAttributes(q, false).isEmpty());

    TreeMultimap<Integer, ITmfStateInterval> countersIntervals = TreeMultimap.create(Comparator.naturalOrder(),
            Comparator.comparingLong(ITmfStateInterval::getStartTime));

    Iterable<@NonNull ITmfStateInterval> query2d = ss.query2D(entries.values(), times);
    for (ITmfStateInterval interval : query2d) {
        if (monitor != null && monitor.isCanceled()) {
            return null;
        }
        countersIntervals.put(interval.getAttribute(), interval);
    }

    ImmutableList.Builder<IYModel> ySeries = ImmutableList.builder();
    for (Entry<Long, Integer> entry : entries.entrySet()) {
        if (monitor != null && monitor.isCanceled()) {
            return null;
        }
        int quark = entry.getValue();
        double[] yValues = buildYValues(countersIntervals.get(quark), filter);
        String seriesName = getTrace().getName() + '/' + ss.getFullAttributePath(quark);
        ySeries.add(new YModel(entry.getKey(), seriesName, yValues));
    }

    return ySeries.build();
}
 

public static <T extends Comparable, E extends Comparable> Iterator<T> extractAndSort(Iterator<T> input, Function<T, E> extractor) {
    TreeMultimap<E, T> reorgnized = TreeMultimap.create();
    while (input.hasNext()) {
        T t = input.next();
        E e = extractor.apply(t);
        reorgnized.put(e, t);
    }
    return reorgnized.values().iterator();
}
 

@Test
public void testDuplicateTerm() throws Throwable {
    List<String> filterSet1 = Lists.newArrayList("a", "b", "c", "q", "r", "s");
    Itr<String> f1 = new Itr<>(filterSet1);
    List<String> filterSet2 = Lists.newArrayList("a", "b", "c", "d", "e", "f");
    Itr<String> f2 = new Itr<>(filterSet2);
    TreeMultimap<String,NestedIterator<String>> mmap = TreeMultimap.create(Util.keyComparator(), Util.hashComparator());
    mmap.put(f1.next(), f1);
    mmap.put(f2.next(), f2);
    
    assertTrue(NegationFilter.isFiltered("c", mmap, Util.keyTransformer()));
    assertEquals(1, mmap.keySet().size());
    assertEquals("c", mmap.keySet().iterator().next());
    assertEquals(2, mmap.values().size());
}
 

@Override
public @NonNull TmfModelResponse<@NonNull TmfTreeModel<@NonNull TimeGraphEntryModel>> fetchTree(@NonNull Map<@NonNull String, @NonNull Object> fetchParameters, @Nullable IProgressMonitor monitor) {
    if (fLastEnd == Long.MAX_VALUE) {
        return new TmfModelResponse<>(new TmfTreeModel<>(Collections.emptyList(), filter(Objects.requireNonNull(fTraceEntry), fTidToEntry, fetchParameters)), ITmfResponse.Status.COMPLETED, CommonStatusMessage.COMPLETED);
    }

    fModule.waitForInitialization();
    ITmfStateSystem ss = fModule.getStateSystem();
    if (ss == null) {
        return new TmfModelResponse<>(null, ITmfResponse.Status.FAILED, CommonStatusMessage.ANALYSIS_INITIALIZATION_FAILED);
    }

    /*
     * As we are caching the intermediate result, we only want a single thread to
     * update them.
     */
    synchronized (fBuildMap) {
        boolean complete = ss.waitUntilBuilt(0);
        @NonNull List<@NonNull TimeGraphEntryModel> list = Collections.emptyList();
        /* Don't query empty state system */
        if (ss.getNbAttributes() > 0 && ss.getStartTime() != Long.MIN_VALUE) {
            long end = ss.getCurrentEndTime();
            fLastEnd = Long.max(fLastEnd, ss.getStartTime());

            TreeMultimap<Integer, ITmfStateInterval> threadData = TreeMultimap.create(Comparator.naturalOrder(),
                    Comparator.comparing(ITmfStateInterval::getStartTime));

            /*
             * Create a List with the threads' PPID and EXEC_NAME quarks for the 2D query .
             */
            List<Integer> quarks = new ArrayList<>(ss.getQuarks(Attributes.THREADS, WILDCARD, Attributes.EXEC_NAME));
            quarks.addAll(ss.getQuarks(Attributes.THREADS, WILDCARD, Attributes.PPID));
            quarks.addAll(ss.getQuarks(Attributes.THREADS, WILDCARD, Attributes.PID));
            try {
                for (ITmfStateInterval interval : ss.query2D(quarks, Long.min(fLastEnd, end), end)) {
                    if (monitor != null && monitor.isCanceled()) {
                        return new TmfModelResponse<>(null, ITmfResponse.Status.CANCELLED, CommonStatusMessage.TASK_CANCELLED);
                    }
                    threadData.put(interval.getAttribute(), interval);
                }
            } catch (TimeRangeException | StateSystemDisposedException e) {
                return new TmfModelResponse<>(null, ITmfResponse.Status.FAILED, e.getClass().getName() + ':' + String.valueOf(e.getMessage()));
            }

            // update the trace Entry.
            TimeGraphEntryModel traceEntry = new TimeGraphEntryModel(fTraceId, -1, getTrace().getName(), ss.getStartTime(), end);
            fTraceEntry = traceEntry;

            for (Integer threadQuark : ss.getQuarks(Attributes.THREADS, WILDCARD)) {
                String threadAttributeName = ss.getAttributeName(threadQuark);
                Pair<Integer, Integer> entryKey = Attributes.parseThreadAttributeName(threadAttributeName);
                int threadId = entryKey.getFirst();
                if (threadId < 0) {
                    // ignore the 'unknown' (-1) thread
                    continue;
                }

                int execNameQuark = ss.optQuarkRelative(threadQuark, Attributes.EXEC_NAME);
                int ppidQuark = ss.optQuarkRelative(threadQuark, Attributes.PPID);
                int pidQuark = ss.optQuarkRelative(threadQuark, Attributes.PID);
                NavigableSet<ITmfStateInterval> ppidIntervals = threadData.get(ppidQuark);
                NavigableSet<ITmfStateInterval> pidIntervals = threadData.get(pidQuark);
                for (ITmfStateInterval execNameInterval : threadData.get(execNameQuark)) {
                    if (monitor != null && monitor.isCanceled()) {
                        return new TmfModelResponse<>(null, ITmfResponse.Status.CANCELLED, CommonStatusMessage.TASK_CANCELLED);
                    }
                    updateEntry(threadQuark, entryKey, ppidIntervals, execNameInterval, pidIntervals);
                }
            }

            fLastEnd = end;

            list = filter(traceEntry, fTidToEntry, fetchParameters);
        }

        for (TimeGraphEntryModel model : list) {
            fEntryMetadata.put(model.getId(), model.getMetadata());
        }

        if (complete) {
            fBuildMap.clear();
            fLastEnd = Long.MAX_VALUE;
            return new TmfModelResponse<>(new TmfTreeModel<>(Collections.emptyList(), list), ITmfResponse.Status.COMPLETED, CommonStatusMessage.COMPLETED);
        }

        return new TmfModelResponse<>(new TmfTreeModel<>(Collections.emptyList(), list), ITmfResponse.Status.RUNNING, CommonStatusMessage.RUNNING);
    }
}
 

public static TreeMultimap<String, byte[]> tokenUpdate(byte[] key, Multimap<String, String> lookup)
		throws InvalidKeyException, InvalidAlgorithmParameterException, NoSuchAlgorithmException,
		NoSuchProviderException, NoSuchPaddingException, IOException {

	// We use a lexicographic sorted list such that the server
	// will not know any information of the inserted elements creation order
	TreeMultimap<String, byte[]> tokenUp = TreeMultimap.create(Ordering.natural(), Ordering.usingToString());

	// Key generation
	for (String word : lookup.keySet()) {

		byte[] key2 = CryptoPrimitivesAndroid.generateCmac(key, 2 + word);
		// generate keys for response-hiding construction for SIV (Synthetic
		// IV)
		byte[] key3 = CryptoPrimitivesAndroid.generateCmac(key, 3 + new String());

		byte[] key4 = CryptoPrimitivesAndroid.generateCmac(key, 4 + word);

		byte[] key5 = CryptoPrimitivesAndroid.generateCmac(key, 5 + word);

		for (String id : lookup.get(word)) {
			int counter = 0;

			if (state.get(word) != null) {
				counter = state.get(word);
			}

			state.put(word, counter + 1);

			byte[] l = CryptoPrimitivesAndroid.generateCmac(key5, "" + counter);

			String value = new String(CryptoPrimitivesAndroid.DTE_encryptAES_CTR_String(key3, key4, id, 20), "ISO-8859-1");

			tokenUp.put(new String(l), CryptoPrimitivesAndroid.encryptAES_CTR_String(key2,
					CryptoPrimitivesAndroid.randomBytes(16), value, sizeOfFileIdentifer));

		}

	}
	return tokenUp;
}
 

@Test
public void testCoercibleGoldfile() {
    TreeMultimap<String, String> coercibleToMap = TreeMultimap.create();
    PDataType[] orderedTypes = PDataTypeFactory.getInstance().getOrderedTypes();
    for (PDataType fromType : orderedTypes) {
        for (PDataType targetType : orderedTypes) {
            if (fromType.isCoercibleTo(targetType)) {
                coercibleToMap.put(fromType.toString(), targetType.toString());
            }
        }
    }
    assertEquals(
          "{BIGINT=[BIGINT, BINARY, DECIMAL, DOUBLE, VARBINARY], "
         + "BIGINT ARRAY=[BIGINT ARRAY, BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, VARBINARY ARRAY], "
         + "BINARY=[BINARY, VARBINARY], "
         + "BINARY ARRAY=[BINARY ARRAY, VARBINARY ARRAY], "
         + "BOOLEAN=[BINARY, BOOLEAN, VARBINARY], "
         + "BOOLEAN ARRAY=[BINARY ARRAY, BOOLEAN ARRAY, VARBINARY ARRAY], "
         + "CHAR=[BINARY, CHAR, VARBINARY, VARCHAR], "
         + "CHAR ARRAY=[BINARY ARRAY, CHAR ARRAY, VARBINARY ARRAY, VARCHAR ARRAY], "
         + "DATE=[BINARY, DATE, TIME, TIMESTAMP, VARBINARY], "
         + "DATE ARRAY=[BINARY ARRAY, DATE ARRAY, TIME ARRAY, TIMESTAMP ARRAY, VARBINARY ARRAY], "
         + "DECIMAL=[DECIMAL, VARBINARY], "
         + "DECIMAL ARRAY=[DECIMAL ARRAY, VARBINARY ARRAY], "
         + "DOUBLE=[BINARY, DECIMAL, DOUBLE, VARBINARY], "
         + "DOUBLE ARRAY=[BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, VARBINARY ARRAY], "
         + "FLOAT=[BINARY, DECIMAL, DOUBLE, FLOAT, VARBINARY], "
         + "FLOAT ARRAY=[BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, FLOAT ARRAY, VARBINARY ARRAY], "
         + "INTEGER=[BIGINT, BINARY, DECIMAL, DOUBLE, FLOAT, INTEGER, VARBINARY], "
         + "INTEGER ARRAY=[BIGINT ARRAY, BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, FLOAT ARRAY, INTEGER ARRAY, VARBINARY ARRAY], "
         + "SMALLINT=[BIGINT, BINARY, DECIMAL, DOUBLE, FLOAT, INTEGER, SMALLINT, VARBINARY], "
         + "SMALLINT ARRAY=[BIGINT ARRAY, BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, FLOAT ARRAY, INTEGER ARRAY, SMALLINT ARRAY, VARBINARY ARRAY], "
         + "TIME=[BINARY, DATE, TIME, TIMESTAMP, VARBINARY], "
         + "TIME ARRAY=[BINARY ARRAY, DATE ARRAY, TIME ARRAY, TIMESTAMP ARRAY, VARBINARY ARRAY], "
         + "TIMESTAMP=[BINARY, TIMESTAMP, VARBINARY], "
         + "TIMESTAMP ARRAY=[BINARY ARRAY, TIMESTAMP ARRAY, VARBINARY ARRAY], "
         + "TINYINT=[BIGINT, BINARY, DECIMAL, DOUBLE, FLOAT, INTEGER, SMALLINT, TINYINT, VARBINARY], "
         + "TINYINT ARRAY=[BIGINT ARRAY, BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, FLOAT ARRAY, INTEGER ARRAY, SMALLINT ARRAY, TINYINT ARRAY, VARBINARY ARRAY], "
         + "UNSIGNED_DATE=[BINARY, DATE, TIME, TIMESTAMP, UNSIGNED_DATE, UNSIGNED_TIME, UNSIGNED_TIMESTAMP, VARBINARY], "
         + "UNSIGNED_DATE ARRAY=[BINARY ARRAY, DATE ARRAY, TIME ARRAY, TIMESTAMP ARRAY, UNSIGNED_DATE ARRAY, UNSIGNED_TIME ARRAY, UNSIGNED_TIMESTAMP ARRAY, VARBINARY ARRAY], "
         + "UNSIGNED_DOUBLE=[BINARY, DECIMAL, DOUBLE, UNSIGNED_DOUBLE, VARBINARY], "
         + "UNSIGNED_DOUBLE ARRAY=[BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, UNSIGNED_DOUBLE ARRAY, VARBINARY ARRAY], "
         + "UNSIGNED_FLOAT=[BINARY, DECIMAL, DOUBLE, FLOAT, UNSIGNED_DOUBLE, UNSIGNED_FLOAT, VARBINARY], "
         + "UNSIGNED_FLOAT ARRAY=[BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, FLOAT ARRAY, UNSIGNED_DOUBLE ARRAY, UNSIGNED_FLOAT ARRAY, VARBINARY ARRAY], "
         + "UNSIGNED_INT=[BIGINT, BINARY, DECIMAL, DOUBLE, FLOAT, INTEGER, UNSIGNED_DOUBLE, UNSIGNED_FLOAT, UNSIGNED_INT, UNSIGNED_LONG, VARBINARY], "
         + "UNSIGNED_INT ARRAY=[BIGINT ARRAY, BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, FLOAT ARRAY, INTEGER ARRAY, UNSIGNED_DOUBLE ARRAY, UNSIGNED_FLOAT ARRAY, UNSIGNED_INT ARRAY, UNSIGNED_LONG ARRAY, VARBINARY ARRAY], "
         + "UNSIGNED_LONG=[BIGINT, BINARY, DECIMAL, DOUBLE, UNSIGNED_DOUBLE, UNSIGNED_LONG, VARBINARY], "
         + "UNSIGNED_LONG ARRAY=[BIGINT ARRAY, BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, UNSIGNED_DOUBLE ARRAY, UNSIGNED_LONG ARRAY, VARBINARY ARRAY], "
         + "UNSIGNED_SMALLINT=[BIGINT, BINARY, DECIMAL, DOUBLE, FLOAT, INTEGER, SMALLINT, UNSIGNED_DOUBLE, UNSIGNED_FLOAT, UNSIGNED_INT, UNSIGNED_LONG, UNSIGNED_SMALLINT, VARBINARY], "
         + "UNSIGNED_SMALLINT ARRAY=[BIGINT ARRAY, BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, FLOAT ARRAY, INTEGER ARRAY, SMALLINT ARRAY, UNSIGNED_DOUBLE ARRAY, UNSIGNED_FLOAT ARRAY, UNSIGNED_INT ARRAY, UNSIGNED_LONG ARRAY, UNSIGNED_SMALLINT ARRAY, VARBINARY ARRAY], "
         + "UNSIGNED_TIME=[BINARY, DATE, TIME, TIMESTAMP, UNSIGNED_DATE, UNSIGNED_TIME, UNSIGNED_TIMESTAMP, VARBINARY], "
         + "UNSIGNED_TIME ARRAY=[BINARY ARRAY, DATE ARRAY, TIME ARRAY, TIMESTAMP ARRAY, UNSIGNED_DATE ARRAY, UNSIGNED_TIME ARRAY, UNSIGNED_TIMESTAMP ARRAY, VARBINARY ARRAY], "
         + "UNSIGNED_TIMESTAMP=[BINARY, DATE, TIME, TIMESTAMP, UNSIGNED_DATE, UNSIGNED_TIME, UNSIGNED_TIMESTAMP, VARBINARY], "
         + "UNSIGNED_TIMESTAMP ARRAY=[BINARY ARRAY, DATE ARRAY, TIME ARRAY, TIMESTAMP ARRAY, UNSIGNED_DATE ARRAY, UNSIGNED_TIME ARRAY, UNSIGNED_TIMESTAMP ARRAY, VARBINARY ARRAY], "
         + "UNSIGNED_TINYINT=[BIGINT, BINARY, DECIMAL, DOUBLE, FLOAT, INTEGER, SMALLINT, TINYINT, UNSIGNED_DOUBLE, UNSIGNED_FLOAT, UNSIGNED_INT, UNSIGNED_LONG, UNSIGNED_SMALLINT, UNSIGNED_TINYINT, VARBINARY], "
         + "UNSIGNED_TINYINT ARRAY=[BIGINT ARRAY, BINARY ARRAY, DECIMAL ARRAY, DOUBLE ARRAY, FLOAT ARRAY, INTEGER ARRAY, SMALLINT ARRAY, TINYINT ARRAY, UNSIGNED_DOUBLE ARRAY, UNSIGNED_FLOAT ARRAY, UNSIGNED_INT ARRAY, UNSIGNED_LONG ARRAY, UNSIGNED_SMALLINT ARRAY, UNSIGNED_TINYINT ARRAY, VARBINARY ARRAY], "
         + "VARBINARY=[BINARY, VARBINARY], "
         + "VARBINARY ARRAY=[BINARY ARRAY, VARBINARY ARRAY], "
         + "VARCHAR=[BINARY, CHAR, VARBINARY, VARCHAR], "
         + "VARCHAR ARRAY=[BINARY ARRAY, CHAR ARRAY, VARBINARY ARRAY, VARCHAR ARRAY]}", 
         coercibleToMap.toString());
}
 

@Test
public void testAggregateFourWithDeepCopy() throws IOException {
    TreeMultimap<Key,Value> map = TreeMultimap.create();
    
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithUid("abc.1").build().toByteArray()));
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithUid("abc.2").build().toByteArray()));
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithUid("abc.3").build().toByteArray()));
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithUid("abc.4").build().toByteArray()));
    
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abd"), new Value(createValueWithUid("abc.3").build().toByteArray()));
    
    SortedMultiMapIterator data = new SortedMultiMapIterator(map);
    
    PropogatingIterator iter = new PropogatingIterator();
    Map<String,String> options = Maps.newHashMap();
    
    options.put(PropogatingIterator.AGGREGATOR_DEFAULT, GlobalIndexUidAggregator.class.getCanonicalName());
    
    IteratorEnvironment env = new MockIteratorEnvironment(false);
    
    iter.init(data, options, env);
    iter = iter.deepCopy(env);
    
    iter.seek(new Range(), Collections.emptyList(), false);
    
    Assert.assertTrue(iter.hasTop());
    
    Key topKey = iter.getTopKey();
    
    Assert.assertEquals(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), topKey);
    validateUids(iter.getTopValue(), "abc.1", "abc.2", "abc.3", "abc.4");
    iter.next();
    topKey = iter.getTopKey();
    Assert.assertEquals(newKey(SHARD, FIELD_TO_AGGREGATE, "abd"), topKey);
    validateUids(iter.getTopValue(), "abc.3");
    
}
 

@Test
public void testForceNoPropogate() throws IOException {
    TreeMultimap<Key,Value> map = TreeMultimap.create();
    
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithUid("abc.0").build().toByteArray()));
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abc"), new Value(createValueWithRemoveUid("abc.0").build().toByteArray()));
    
    map.put(newKey(SHARD, FIELD_TO_AGGREGATE, "abd"), new Value(createValueWithUid("abc.3").build().toByteArray()));
    
    SortedMultiMapIterator data = new SortedMultiMapIterator(map);
    
    PropogatingIterator iter = new PropogatingIterator();
    Map<String,String> options = Maps.newHashMap();
    
    options.put(PropogatingIterator.AGGREGATOR_DEFAULT, GlobalIndexUidAggregator.class.getCanonicalName());
    
    IteratorEnvironment env = new MockIteratorEnvironment(true);
    
    iter.init(data, options, env);
    
    iter.seek(new Range(), Collections.emptyList(), false);
    
    Assert.assertTrue(iter.hasTop());
    
    Key topKey = iter.getTopKey();
    
    topKey = iter.getTopKey();
    Assert.assertEquals(newKey(SHARD, FIELD_TO_AGGREGATE, "abd"), topKey);
    
}
 

public static TreeMultimap<String, byte[]> tokenUpdate(byte[] key, Multimap<String, String> lookup)
		throws InvalidKeyException, InvalidAlgorithmParameterException, NoSuchAlgorithmException,
		NoSuchProviderException, NoSuchPaddingException, IOException {

	// We use a lexicographic sorted list such that the server
	// will not know any information of the inserted elements creation order
	TreeMultimap<String, byte[]> tokenUp = TreeMultimap.create(Ordering.natural(), Ordering.usingToString());
	// Key generation

	SecureRandom random = new SecureRandom();
	random.setSeed(CryptoPrimitives.randomSeed(16));
	byte[] iv = new byte[16];

	for (String word : lookup.keySet()) {

		byte[] key1 = CryptoPrimitives.generateCmac(key, 1 + new String());

		byte[] key2 = CryptoPrimitives.generateCmac(key, 2 + word);

		for (String id : lookup.get(word)) {

			random.nextBytes(iv);

			int counter = 0;

			if (state.get(word) != null) {
				counter = state.get(word);
			}

			state.put(word, counter + 1);

			byte[] l = CryptoPrimitives.generateCmac(key2, "" + counter);

			byte[] value = CryptoPrimitives.encryptAES_CTR_String(key1, iv, id, sizeOfFileIdentifer);
			tokenUp.put(new String(l), value);
		}

	}
	return tokenUp;
}