com.google.common.hash.HashFunction Java Examples

/**
 * Create an analyzer to calculate a minhash.
 * 
 * @param tokenizer a tokenizer to parse a text
 * @param hashBit the number of hash bits
 * @param seed a base seed for hash function
 * @param num the number of hash functions
 * @return analyzer used by {@link MinHash#calculate(Analyzer, String)}
 */
public static Analyzer createAnalyzer(final Tokenizer tokenizer,
        final int hashBit, final int seed, final int num) {
    final HashFunction[] hashFunctions = MinHash.createHashFunctions(seed,
            num);
    final Analyzer minhashAnalyzer = new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(
                final String fieldName) {
            final TokenStream stream = new MinHashTokenFilter(
                    tokenizer, hashFunctions, hashBit);
            return new TokenStreamComponents(tokenizer, stream);
        }
    };
    return minhashAnalyzer;
}
 

public static HashFunction getHasher(HashType hashType) {
  switch(hashType) {
    case MURMUR3_128:
      return Hashing.murmur3_128();
    case MURMUR3_32:
      return Hashing.murmur3_32();
    case SIPHASH24:
      return Hashing.sipHash24();
    case MD5:
      return Hashing.md5();
    case SHA1:
      return Hashing.sha1();
    case SHA256:
      return Hashing.sha256();
    case SHA512:
      return Hashing.sha512();
    case ADLER32:
      return Hashing.adler32();
    case CRC32:
      return Hashing.crc32();
    case CRC32C:
      return Hashing.crc32c();
    default:
      throw new IllegalArgumentException(Utils.format("Unsupported Hashing Algorithm: {}", hashType.name()));
  }
}
 

private int hash() {
    // Guava documentation recommends using putUnencodedChars to hash raw character bytes within any encoding
    // unless cross-language compatibility is needed. See the Hasher.putString documentation for more info.
    Funnel<TrafficSelector> selectorFunnel = (from, into) -> from.criteria()
            .forEach(c -> into.putUnencodedChars(c.toString()));

    HashFunction hashFunction = Hashing.murmur3_32();
    HashCode hashCode = hashFunction.newHasher()
            .putUnencodedChars(deviceId.toString())
            .putObject(selector, selectorFunnel)
            .putInt(priority)
            .putUnencodedChars(tableId.toString())
            .hash();

    return hashCode.asInt();
}
 

public void apply(String idProcess, ParameterTransformation parameterTransformation, ObjectNode jsonValue) {
    JsonNode valueField = at(parameterTransformation.getProcessHashData().getField(), jsonValue);
    if(valueField != null &&
            StringUtils.isNotBlank(valueField.asText())){
        switch (parameterTransformation.getProcessHashData().getTypeHash()){
            case SHA256:
                HashFunction m_hash256 = Hashing.sha256();
                String valueHash256 = m_hash256.hashBytes(valueField.asText().getBytes()).toString();
                put(jsonValue, parameterTransformation.getProcessHashData().getField(), valueHash256);
                break;
            case MURMUR3:
                HashFunction m_hashMurmur3 = Hashing.murmur3_128();
                String valueHashMurmur3 = m_hashMurmur3.hashBytes(valueField.asText().getBytes()).toString();
                put(jsonValue, parameterTransformation.getProcessHashData().getField(), valueHashMurmur3);
                break;
            default:
                log.error("Type Hash not manage {}",parameterTransformation.getProcessHashData().getTypeHash());
                break;

        }
    }
}
 

private static HashFunction configureHash(Algorithm alg, long seedNSalt, long addlSipSeed) {
	switch (alg) {
	case xxHash64:
		return new xxHashFunction(seedNSalt);
	case Murmur3_128:
		return Hashing.murmur3_128((int) seedNSalt);
	case Murmur3_32:
		return Hashing.murmur3_32((int) seedNSalt);
	case sha256:
		return Hashing.sha1();
	case sipHash24:
		return Hashing.sipHash24(seedNSalt, addlSipSeed);
	default:
		throw new IllegalArgumentException("Invalid Enum Hashing Algorithm???");
	}
}
 

@Test
public void testHashCodesM3_128_longs() {
  int seed = 123;
  Random rand = new Random(seed);
  HashFunction hf = Hashing.murmur3_128(seed);
  for (int i = 0; i < 1000; i++) {
    long val = rand.nextLong();
    byte[] data = ByteBuffer.allocate(8).putLong(val).array();
    // guava stores the hashcodes in little endian order
    ByteBuffer buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN);
    buf.put(hf.hashBytes(data).asBytes());
    buf.flip();
    long gl1 = buf.getLong();
    long gl2 = buf.getLong(8);
    long[] hc = Murmur3.hash128(data, data.length, seed);
    long m1 = hc[0];
    long m2 = hc[1];
    assertEquals(gl1, m1);
    assertEquals(gl2, m2);
  }
}
 

/**
 * Returns a unique File for the pre-dexed library, even
 * if there are 2 libraries with the same file names (but different
 * paths)
 * <p>
 * If multidex is enabled the return File is actually a folder.
 *
 * @param outFolder the output folder.
 * @param inputFile the library.
 */
@NonNull
static File getDexFileName(@NonNull File outFolder, @NonNull File inputFile) {
    // get the filename
    String name = inputFile.getName();
    // remove the extension
    int pos = name.lastIndexOf('.');
    if (pos != -1) {
        name = name.substring(0, pos);
    }

    // add a hash of the original file path.
    String input = inputFile.getAbsolutePath();
    HashFunction hashFunction = Hashing.sha1();
    HashCode hashCode = hashFunction.hashString(input, Charsets.UTF_16LE);

    return new File(outFolder, name + "-" + hashCode.toString() + SdkConstants.DOT_JAR);
}
 

public boolean isFullDuplicate(IHttpRequestResponse messageInfo) {
    PrintWriter stdout = new PrintWriter(callbacks.getStdout(), true);
    IResponseInfo respInfo = helpers.analyzeResponse(messageInfo.getResponse());

    if (dubBloomFilter == null) return false;

    HashFunction m_hash = Hashing.murmur3_32();
    if (helpers.bytesToString(messageInfo.getResponse()).length() > respInfo.getBodyOffset()) {
        String body = helpers.bytesToString(messageInfo.getResponse()).substring(respInfo.getBodyOffset());

        /* full-dub detection */
        String dedupHashValue = m_hash.hashBytes(helpers.stringToBytes(body)).toString();
        if (dubBloomFilter.mightContain(dedupHashValue)) {
            return true;
        }
        dubBloomFilter.put(dedupHashValue);
    }

    return false;
}
 

private String generateHash(
    Record record,
    HashType hashType,
    Collection<String> fieldsToHash,
    boolean includeRecordHeader,
    boolean useSeparator,
    char separator
) throws StageException {
  try {
    HashFunction hasher = HashingUtil.getHasher(hashType.getHashType());
    HashingUtil.RecordFunnel recordFunnel = HashingUtil.getRecordFunnel(
        fieldsToHash,
        includeRecordHeader,
        useSeparator,
        separator
    );
    return hasher.hashObject(record, recordFunnel).toString();
  } catch (IllegalArgumentException e) {
    throw new OnRecordErrorException(Errors.HASH_00, hashType.getDigest(), e.toString(), e);
  }
}
 

public final static long[] computeSequenceHashesLong(final String seq, final int nGramSize, final int seed, final boolean doReverseCompliment)
{
	HashFunction hf = Hashing.murmur3_128(seed);

	long[] hashes = new long[seq.length() - nGramSize + 1];
	for (int iter = 0; iter < hashes.length; iter++)
	{
		String str = seq.substring(iter, iter + nGramSize);
		String strReverse = null;
		if (doReverseCompliment)
		{
			strReverse  = Utils.rc(str);
			if (strReverse.compareTo(str)<0)
				str = strReverse;
		}
		
		HashCode hc = hf.newHasher().putUnencodedChars(str).hash();
		hashes[iter] = hc.asLong();
	}

	return hashes;
}
 

public final static int[] computeSequenceHashes(final String seq, final int nGramSize, boolean doReverseCompliment)
{
	HashFunction hf = Hashing.murmur3_32(0);

	int[] hashes = new int[seq.length() - nGramSize + 1];
	for (int iter = 0; iter < hashes.length; iter++)
	{
		String str = seq.substring(iter, iter + nGramSize);
		
		String strReverse = null;
		if (doReverseCompliment)
		{
			strReverse  = Utils.rc(str);
			if (strReverse.compareTo(str)<0)
				str = strReverse;
		}

		HashCode hc = hf.newHasher().putUnencodedChars(str).hash();
		hashes[iter] = hc.asInt();
	}

	return hashes;
}
 

@Inject
@VisibleForTesting
public EntrySerializerImpl(
    @MaxEntrySize Amount<Integer, Data> maxEntrySize,
    @LogEntryHashFunction HashFunction hashFunction) {

  this.hashFunction = requireNonNull(hashFunction);
  maxEntrySizeBytes = maxEntrySize.as(Data.BYTES);
}
 

ConsistentHash(HashFunction hashFunction,
               int numOfReplicas,
               StatsReceiver statsReceiver) {
    this.hashFunction = hashFunction;
    this.numOfReplicas = numOfReplicas;
    this.circle = new TreeMap<Long, SocketAddress>();

    this.hostAddedCounter = statsReceiver.counter0("adds");
    this.hostRemovedCounter = statsReceiver.counter0("removes");
}
 

@Test
@Ignore
public void testPrecise() throws CardinalityMergeException
{
    int cardinality = 1000000000;
    int b = 12;
    LogLog baseline = new LogLog(b);
    LogLog guava128 = new LogLog(b);
    HashFunction hf128 = Hashing.murmur3_128();
    for (int j = 0; j < cardinality; j++)
    {
        Double val = Math.random();
        String valString = val.toString();
        baseline.offer(valString);
        guava128.offerHashed(hf128.hashString(valString).asLong());
        if (j > 0 && j % 1000000 == 0)
        {
            System.out.println("current count: " + j);
        }
    }


    long baselineEstimate = baseline.cardinality();
    long g128Estimate = guava128.cardinality();
    double se = cardinality * (1.04 / Math.sqrt(Math.pow(2, b)));
    double baselineError = (baselineEstimate-cardinality)/(double)cardinality;
    double g128Error = (g128Estimate-cardinality)/(double)cardinality;
    System.out.format("b: %f g128 %f", baselineError, g128Error);
    assertTrue("baseline estimate bigger than expected", baselineEstimate >= cardinality - (2 * se));
    assertTrue("baseline estimate smaller than expected", baselineEstimate <= cardinality + (2 * se));
    assertTrue("g128 estimate bigger than expected", g128Estimate >= cardinality - (2 * se));
    assertTrue("g128 estimate smaller than expected", g128Estimate <= cardinality + (2 * se));
}
 

@Provides
@Singleton
@Named("dao")
AuthIdentityManager<ApiKey> provideAuthIdentityManagerDAO(
        AuthorizationConfiguration config, DataStore dataStore, @ApiKeyHashFunction HashFunction hash,
        @IdentityIdSupplier Supplier<String> identityIdSupplier, @SystemTablePlacement String tablePlacement) {
    return new TableAuthIdentityManagerDAO<>(ApiKey.class, dataStore, config.getIdentityTable(),
            config.getIdIndexTable(), tablePlacement, identityIdSupplier, hash);
}
 

public ConsistentHash(HashFunction hashFunction, Funnel<K> keyFunnel, Funnel<N> nodeFunnel, Collection<N> nodes) {
	this.hashFunction = hashFunction;
	this.nodeFunnel = nodeFunnel;
	this.keyFunnel = keyFunnel;
	for (N node : nodes) {
		add(node);
	}
}
 

private static byte[] download(String path, String hash, IntConsumer progress) throws IOException, VerificationException
{
	HashFunction hashFunction = Hashing.sha256();
	Hasher hasher = hashFunction.newHasher();
	ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();

	URL url = new URL(path);
	HttpURLConnection conn = (HttpURLConnection) url.openConnection();
	conn.setRequestProperty("User-Agent", USER_AGENT);
	conn.getResponseCode();

	InputStream err = conn.getErrorStream();
	if (err != null)
	{
		err.close();
		throw new IOException("Unable to download " + path + " - " + conn.getResponseMessage());
	}

	int downloaded = 0;
	try (InputStream in = conn.getInputStream())
	{
		int i;
		byte[] buffer = new byte[1024 * 1024];
		while ((i = in.read(buffer)) != -1)
		{
			byteArrayOutputStream.write(buffer, 0, i);
			hasher.putBytes(buffer, 0, i);
			downloaded += i;
			progress.accept(downloaded);
		}
	}

	HashCode hashCode = hasher.hash();
	if (!hash.equals(hashCode.toString()))
	{
		throw new VerificationException("Unable to verify resource " + path + " - expected " + hash + " got " + hashCode.toString());
	}

	return byteArrayOutputStream.toByteArray();
}
 

@Override
public int hashCode() {
    HashFunction hf = Hashing.goodFastHash(32);
    Hasher h = hf.newHasher();
    h.putInt(slots.size());
    for (int i=0; i<slots.size(); i++) {
        h.putInt(slots.get(i).size());
        for (int j=0; j<slots.size(); j++) {
            h.putBytes(slots.get(i).get(j).getLowerRange());
            h.putBytes(slots.get(i).get(j).getUpperRange());
        }
    }
    return h.hash().asInt();
}
 

public final static int[] computeHashesIntString(String obj, int numWords, int seed)
{
	int[] hashes = new int[numWords];

	HashFunction hf = Hashing.murmur3_32(seed);

	for (int iter = 0; iter < numWords; iter++)
	{
		HashCode hc = hf.newHasher().putUnencodedChars(obj).putInt(iter).hash();

		hashes[iter] = hc.asInt();
	}

	return hashes;
}
 

/**
 * Hashes the contents of this byte source using the given hash function.
 *
 * @throws IOException if an I/O error occurs in the process of reading from this source
 */


public HashCode hash(HashFunction hashFunction) throws IOException {
  Hasher hasher = hashFunction.newHasher();
  copyTo(Funnels.asOutputStream(hasher));
  return hasher.hash();
}
 

@Override
public int hashCode() {
    HashFunction hf = Hashing.goodFastHash(32);
    Hasher h = hf.newHasher();
    h.putInt(slots.size());
    for (int i=0; i<slots.size(); i++) {
        h.putInt(slots.get(i).size());
        for (int j=0; j<slots.size(); j++) {
            h.putBytes(slots.get(i).get(j).getLowerRange());
            h.putBytes(slots.get(i).get(j).getUpperRange());
        }
    }
    return h.hash().asInt();
}
 

@Test
public void testHashCodesM3_32_string() {
  String key = "test";
  int seed = 123;
  HashFunction hf = Hashing.murmur3_32(seed);
  int hc1 = hf.hashBytes(key.getBytes()).asInt();
  int hc2 = Murmur3.hash32(key.getBytes(), key.getBytes().length, seed);
  assertEquals(hc1, hc2);

  key = "testkey";
  hc1 = hf.hashBytes(key.getBytes()).asInt();
  hc2 = Murmur3.hash32(key.getBytes(), key.getBytes().length, seed);
  assertEquals(hc1, hc2);
}
 

/** Return the hash function corresponding to a given message-digest algorithm given by name.
 *
 * @param messageDigest a message-digest algorithm (e.g., <code>MurmurHash3</code> or <code>MD5</code>); {@code null} if {@code messageDigest} is the empty string.
 */
@SuppressWarnings("deprecation")
public final static HashFunction forName(final String messageDigest) throws NoSuchAlgorithmException {
	if ("".equals(messageDigest)) return null;
	if ("MD5".equalsIgnoreCase(messageDigest)) return Hashing.md5();
	if ("MurmurHash3".equalsIgnoreCase(messageDigest)) return Hashing.murmur3_128();
	throw new NoSuchAlgorithmException("Unknown hash function " + messageDigest);
}
 

public static HashFunction forName(String name) {
    switch (name) {
        case "murmur3_32":
            return Hashing.murmur3_32();
        case "murmur3_128":
            return Hashing.murmur3_128();
        case "crc32":
            return Hashing.crc32();
        case "md5":
            return Hashing.md5();
        default:
            throw new IllegalArgumentException("Can't find hash function with name " + name);
    }
}
 

private static Function<File, String> hashWith(Supplier<HashFunction> algorithm) {
    return (file) -> {
        try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) {
            Hasher hasher = algorithm.get().newHasher();
            int b;
            while ((b = inputStream.read()) != -1) {
                hasher.putByte((byte) b);
            }
            return hasher.hash().toString();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    };
}
 

/**
 * Hashes the contents of this byte source using the given hash function.
 *
 * @throws IOException if an I/O error occurs in the process of reading from this source
 */


public HashCode hash(HashFunction hashFunction) throws IOException {
  Hasher hasher = hashFunction.newHasher();
  copyTo(Funnels.asOutputStream(hasher));
  return hasher.hash();
}
 

/** Get the hash function for this type of hash */
public HashFunction getHashFunction() {
  if (sha1OrSha256.isLeft()) {
    return Hashing.sha1();
  } else {
    return Hashing.sha256();
  }
}
 

@Test
public void checkUniformDistribution() {
    final long samples = 100000L;
    final int rounds = 5;
    final double percentMarginOfError = 0.5;
    final int numKeys = 100;
    final Random random = new Random();

    for (int round = 0; round < rounds; round++) {
        logger.info(String.format("checkUniformDistribution - round %s: %d samples", round + 1, samples));

        for (final HashFunction hash: new HashFunction[]{md5(), murmur3_128(), sipHash24(), sha256()}) {
            long sum = 0L;
            final long initialTime = System.currentTimeMillis();
            for (Integer counter = 0; counter < samples; counter++) {
                final int chosen = (int) (random.nextFloat() * (numKeys - Float.MIN_VALUE));
                sum += Hashing.consistentHash(hash.hashInt(chosen), (int) numBackends);
            }

            final long finishTime = System.currentTimeMillis();

            final double result = (numBackends * (numBackends - 1) / 2.0) * (samples / numBackends);

            logger.info(String.format("-> checkUniformDistribution (%s): Time spent (ms): %d. NonUniformDistRatio (smaller is better): %.4f%%",
                    hash, finishTime - initialTime, Math.abs(100.0 * (result-sum) / result)));

            final double topLimit = sum * (1.0 + percentMarginOfError);
            final double bottomLimit = sum * (1.0 - percentMarginOfError);

            try {
                assertThat(result).isGreaterThanOrEqualTo(bottomLimit).isLessThanOrEqualTo(topLimit);
            } catch (AssertionError e) {
                logger.error("Error when testing " + hash);
                throw e;
            }
        }
    }
}
 

/**
 * Hashes the contents of this byte source using the given hash function.
 *
 * @throws IOException if an I/O error occurs in the process of reading from this source
 */


public HashCode hash(HashFunction hashFunction) throws IOException {
  Hasher hasher = hashFunction.newHasher();
  copyTo(Funnels.asOutputStream(hasher));
  return hasher.hash();
}
 

public MinHashTokenFilter(final TokenStream input,
        final HashFunction[] hashFunctions, final int hashBit) {
    super(input);
    this.hashFunctions = hashFunctions;
    this.hashBit = hashBit;
    minHashValues = new long[hashFunctions.length];
}