Java Code Examples for com.google.common.hash.Hashing#md5()

@Before
public void mySetUp() throws Exception {
  control = createControl();
  addTearDown(() -> {
    if (mainException.get().isPresent()) {
      RuntimeException e = mainException.get().get();
      LOG.error("Scheduler main exited with an exception", e);
      fail(e.getMessage());
    }
    control.verify();
  });
  backupDir = temporaryFolder.newFolder();
  driver = control.createMock(SchedulerDriver.class);
  // This is necessary to allow driver to block, otherwise it would stall other mocks.
  EasyMock.makeThreadSafe(driver, false);

  driverFactory = control.createMock(DriverFactory.class);
  log = control.createMock(Log.class);
  logStream = control.createMock(Stream.class);
  streamMatcher = LogOpMatcher.matcherFor(logStream);
  entrySerializer = new EntrySerializer.EntrySerializerImpl(
      Amount.of(512, Data.KB),
      Hashing.md5());
}
 

@Override
protected void configure() {
  bind(new TypeLiteral<Amount<Integer, Data>>() { }).annotatedWith(MaxEntrySize.class)
      .toInstance(options.maxLogEntrySize);
  bind(LogManager.class).in(Singleton.class);
  bind(LogPersistence.class).in(Singleton.class);
  bind(Persistence.class).to(LogPersistence.class);
  expose(Persistence.class);
  expose(LogPersistence.class);

  bind(EntrySerializer.class).to(EntrySerializerImpl.class);
  // TODO(ksweeney): We don't need a cryptographic checksum here - assess performance of MD5
  // versus a faster error-detection checksum like CRC32 for large Snapshots.
  @SuppressWarnings("deprecation")
  HashFunction hashFunction = Hashing.md5();
  bind(HashFunction.class).annotatedWith(LogEntryHashFunction.class).toInstance(hashFunction);

  bind(SnapshotDeduplicator.class).to(SnapshotDeduplicatorImpl.class);

  install(new FactoryModuleBuilder()
      .implement(StreamManager.class, StreamManagerImpl.class)
      .build(StreamManagerFactory.class));
}
 

/**
 * Returns the HashFunction named by algorithm
 *
 * See the Hasher class docs for a list of algorithms and guidance on selection.
 *
 * @param algorithm the hash algorithm to use
 * @throws IllegalArgumentException for an invalid seed given the algorithm
 * @throws RuntimeException when the seed cannot be parsed
 */
private void makeHashFunc(String algorithm) throws IllegalArgumentException, RuntimeException
{
  if (hash_func != null) { throw new RuntimeException("The hash function should only be set once per instance"); }

  if (algorithm.startsWith("good-")) {
    int bits = Integer.parseInt(algorithm.substring(5));
    hash_func = Hashing.goodFastHash(bits);
  }
  else if (algorithm.equals("murmur3-32")) { hash_func = Hashing.murmur3_32();  }
  else if (algorithm.equals("murmur3-128")){ hash_func = Hashing.murmur3_128(); }
  else if (algorithm.equals("sip24"))      { hash_func = Hashing.sipHash24();   }
  else if (algorithm.equals("sha1"))       { hash_func = Hashing.sha1();        }
  else if (algorithm.equals("sha256"))     { hash_func = Hashing.sha256();      }
  else if (algorithm.equals("sha512"))     { hash_func = Hashing.sha512();      }
  else if (algorithm.equals("md5"))        { hash_func = Hashing.md5();         }
  else if (algorithm.equals("adler32"))    { hash_func = Hashing.adler32();     }
  else if (algorithm.equals("crc32"))      { hash_func = Hashing.crc32();       }
  else { throw new IllegalArgumentException("No hash function found for algorithm "+algorithm+". Allowed values include "+HASH_NAMES); }
}
 

/**
 * Distribute triggers on nodes using a consistent hashing strategy.
 * This strategy allows to scale and minimize changes and re-distribution when cluster changes.
 *
 * @param entries a list of entries to distribute
 * @param buckets a table of nodes
 * @return a map of entries distributed across nodes
 */
public Map<PartitionEntry, Integer> calculatePartition(List<PartitionEntry> entries,
                                                       Map<Integer, Integer> buckets) {
    if (entries == null) {
        throw new IllegalArgumentException("entries must be not null");
    }
    if (isEmpty(buckets)) {
        throw new IllegalArgumentException("entries must be not null");
    }
    HashFunction md5 = Hashing.md5();
    int numBuckets = buckets.size();
    Map<PartitionEntry, Integer> newPartition = new HashMap<>();
    for (PartitionEntry entry : entries) {
        newPartition.put(entry, buckets.get(Hashing.consistentHash(md5.hashInt(entry.hashCode()), numBuckets)));
    }
    return newPartition;
}
 

@Test
public void testWriteAndReadDeflatedEntry() throws Exception {
  Snapshot snapshot = createSnapshot();
  LogEntry snapshotLogEntry = LogEntry.snapshot(snapshot);
  LogEntry deflatedSnapshotEntry = Entries.deflate(
      LogEntry.deduplicatedSnapshot(new SnapshotDeduplicatorImpl().deduplicate(snapshot)));

  Entry snapshotEntry = createMock(Entry.class);
  expect(stream.append(entryEq(deflatedSnapshotEntry))).andReturn(position1);
  stream.truncateBefore(position1);

  expect(snapshotEntry.contents()).andReturn(encode(deflatedSnapshotEntry));

  expect(stream.readAll()).andReturn(ImmutableList.of(snapshotEntry).iterator());

  control.replay();

  HashFunction md5 = Hashing.md5();
  StreamManagerImpl streamManager = new StreamManagerImpl(
      stream,
      new EntrySerializer.EntrySerializerImpl(NO_FRAMES_EVER_SIZE, md5),
      md5,
      new SnapshotDeduplicatorImpl());
  streamManager.snapshot(snapshot);
  assertEquals(
      ImmutableList.of(snapshotLogEntry),
      ImmutableList.copyOf(streamManager.readFromBeginning()));
}
 

private StreamManager createStreamManager(final Amount<Integer, Data> maxEntrySize) {
  return new StreamManagerImpl(
      stream,
      new EntrySerializer.EntrySerializerImpl(maxEntrySize, Hashing.md5()),
      Hashing.md5(),
      new SnapshotDeduplicatorImpl());
}
 

/**
 * Set the input data the given Compression. Will automatically
 * set calculate the md5 sum and length properties
 * @param in InputStream
 * @return Builder
 * @throws IOException
 */
public Builder data(
  InputStream in, 
  Compression<?,?> compression) 
    throws IOException {
  StringWriter writer = new StringWriter();
  OutputStream out = 
    BaseEncoding.base64Url().encodingStream(writer);
  if (compression != null)
    out = compression.compressor(out);
  HashingOutputStream hout = 
    new HashingOutputStream(
      Hashing.md5(), out);
  byte[] buf = new byte[1024];
  int r = -1;
  long size = 0;
  while((r = in.read(buf)) > -1) {
    hout.write(buf,0,r);
    size += r;
  }
  set("length", size);
  if (compression != null) {
    set("compression", compression.label());
    compression.finish(out);
  }
  hout.close();
  set("md5", hout.hash().toString());
  return set("data",writer.toString());
}
 

public static Optional<String> fileHash(Optional<VirtualFile> vf) {
  if (!vf.isPresent()) {
    return Optional.empty();
  }
  HashFunction hf = Hashing.md5();
  try {
    byte[] bytes = Files.readAllBytes(Paths.get(vf.get().getPath()));
    HashCode hash = hf.newHasher().putBytes(bytes).hash();
    return Optional.of(hash.toString());
  }
  catch (IOException e) {
    e.printStackTrace();
    return Optional.empty();
  }
}
 

public static String readAllAndgetMD5 (InputStream in) throws IOException
{
	com.google.common.hash.HashingInputStream his = null ;
	try
	{
		his = new com.google.common.hash.HashingInputStream (Hashing.md5(), in) ;
		
		final int bufferSize = 2097152 ;
		final ReadableByteChannel inputChannel = Channels.newChannel(his);
		final ByteBuffer buffer = ByteBuffer.allocateDirect(bufferSize);
		while (inputChannel.read(buffer) != -1) {
			buffer.clear();
		}
		/*
		byte[] bytesBuffer = new byte[bufferSize] ;
		int r = his.read(bytesBuffer, 0, bufferSize) ;
		while (r != -1)
			r = his.read(bytesBuffer) ;
		*/
		HashCode hc = his.hash() ;
		return (hc != null) ? (hc.toString()) : (null) ;
	}
	finally
	{
		if (his != null)
			his.close() ;
	}
}
 

/**
 * Distribute a new entry across buckets using a consistent hashing strategy.
 *
 * @param newEntry the new entry to distribute
 * @param buckets a table of nodes
 * @return a code of the node which the new entry is placed
 */
public Integer calculateNewEntry(PartitionEntry newEntry, Map<Integer, Integer> buckets) {
    if (newEntry == null) {
        throw new IllegalArgumentException("newEntry must be not null");
    }
    if (isEmpty(buckets)) {
        throw new IllegalArgumentException("buckets must be not null");
    }
    HashFunction md5 = Hashing.md5();
    int numBuckets = buckets.size();
    return buckets.get(Hashing.consistentHash(md5.hashInt(newEntry.hashCode()), numBuckets));
}
 

public int getNodeIndex(int numNodes, String key)
{
  HashFunction hf = Hashing.md5();
  HashCode hc = hf.hashString(key, Charsets.UTF_8);
  int initialNodeIndex = Hashing.consistentHash(hc, numNodes);
  int finalNodeIndex = initialNodeIndex;
  if (hc.asInt() % 2 == 0) {
    finalNodeIndex = getNextRunningNodeIndex(initialNodeIndex);
  }
  else {
    finalNodeIndex = getPreviousRunningNodeIndex(initialNodeIndex);
  }

  return finalNodeIndex;
}
 

/** Return the hash function corresponding to a given message-digest algorithm given by name.
 *
 * @param messageDigest a message-digest algorithm (e.g., <code>MurmurHash3</code> or <code>MD5</code>); {@code null} if {@code messageDigest} is the empty string.
 */
@SuppressWarnings("deprecation")
public final static HashFunction forName(final String messageDigest) throws NoSuchAlgorithmException {
	if ("".equals(messageDigest)) return null;
	if ("MD5".equalsIgnoreCase(messageDigest)) return Hashing.md5();
	if ("MurmurHash3".equalsIgnoreCase(messageDigest)) return Hashing.murmur3_128();
	throw new NoSuchAlgorithmException("Unknown hash function " + messageDigest);
}
 

public static HashFunction forName(String name) {
    switch (name) {
        case "murmur3_32":
            return Hashing.murmur3_32();
        case "murmur3_128":
            return Hashing.murmur3_128();
        case "crc32":
            return Hashing.crc32();
        case "md5":
            return Hashing.md5();
        default:
            throw new IllegalArgumentException("Can't find hash function with name " + name);
    }
}
 

private String getHashFor(File inputFile) {
    String retval = alreadyChecked.get(inputFile.getAbsolutePath());
    if (retval != null) return retval;
    // add a hash of the original file path
    try {
        HashFunction hashFunction = Hashing.md5();
        HashCode hashCode = hashFunction.hashBytes(Files.readAllBytes(inputFile.toPath()));
        retval = hashCode.toString();
        alreadyChecked.put(inputFile.getAbsolutePath(), retval);
        return retval;
    } catch (IOException e) {
        e.printStackTrace();
        return "ERROR";
    }
}
 

/**
 * Returns the hardware properties defined in
 * {@link AvdManager#HARDWARE_INI} as a {@link Map}.
 *
 * This is intended to be dumped in the config.ini and already contains
 * the device name, manufacturer and device hash.
 *
 * @param d The {@link Device} from which to derive the hardware properties.
 * @return A {@link Map} of hardware properties.
 */
@NonNull
public static Map<String, String> getHardwareProperties(@NonNull Device d) {
    Map<String, String> props = getHardwareProperties(d.getDefaultState());
    for (State s : d.getAllStates()) {
        if (s.getKeyState().equals(KeyboardState.HIDDEN)) {
            props.put("hw.keyboard.lid", getBooleanVal(true));
        }
    }

    HashFunction md5 = Hashing.md5();
    Hasher hasher = md5.newHasher();

    ArrayList<String> keys = new ArrayList<String>(props.keySet());
    Collections.sort(keys);
    for (String key : keys) {
        if (key != null) {
            hasher.putString(key, Charsets.UTF_8);
            String value = props.get(key);
            hasher.putString(value == null ? "null" : value, Charsets.UTF_8);
        }
    }
    // store the hash method for potential future compatibility
    String hash = "MD5:" + hasher.hash().toString();
    props.put(AvdManager.AVD_INI_DEVICE_HASH_V2, hash);
    props.remove(AvdManager.AVD_INI_DEVICE_HASH_V1);

    props.put(AvdManager.AVD_INI_DEVICE_NAME, d.getId());
    props.put(AvdManager.AVD_INI_DEVICE_MANUFACTURER, d.getManufacturer());
    return props;
}
 

public static String getHashedKey(String key, String hashingAlgorithm) {
    final long start = System.nanoTime();
    HashFunction hf = null; 
    switch(hashingAlgorithm.toLowerCase()) {
        case "murmur3" :
            hf = Hashing.murmur3_128();
            break;

        case "adler32" :
            hf = Hashing.adler32();
            break;

        case "crc32" :
            hf = Hashing.crc32();
            break;

        case "sha1" :
            hf = Hashing.sha1();
            break;

        case "sha256" :
            hf = Hashing.sha256();
            break;

        case "siphash24" :
            hf = Hashing.sipHash24();
            break;

        case "goodfasthash" :
            hf = Hashing.goodFastHash(128);
            break;

        case "md5" :
        default :
            hf = Hashing.md5();
            break;
    }

    final HashCode hc = hf.newHasher().putString(key, Charsets.UTF_8).hash();
    final byte[] digest = hc.asBytes();
    if(log.isDebugEnabled()) log.debug("Key : " + key +"; digest length : " + digest.length + "; byte Array contents : " + Arrays.toString(digest) );
    final String hKey = encoder.encodeToString(digest);
    if(log.isDebugEnabled()) log.debug("Key : " + key +"; Hashed & encoded key : " + hKey + "; Took " + (System.nanoTime() - start) + " nanos");
    return hKey;
}
 

private static HashCode md5HashCode(String input) {
  @SuppressWarnings("deprecation")
  HashFunction hf = Hashing.md5();
  return hf.newHasher().putString(input, Charsets.UTF_8).hash();
}
 

/**
 * Filter the given container response
 *
 * @param containerResponse the response to use
 */
public void doFilter(GenericContainerResponse containerResponse) {

  // get entity of the response
  Object entity = containerResponse.getEntity();

  // no entity, skip
  if (entity == null) {
    return;
  }

  // Only handle JSON content
  if (!MediaType.APPLICATION_JSON_TYPE.equals(containerResponse.getContentType())) {
    return;
  }

  // Get the request
  ApplicationContext applicationContext = ApplicationContext.getCurrent();
  Request request = applicationContext.getRequest();

  // manage only GET requests
  if (!HttpMethod.GET.equals(request.getMethod())) {
    return;
  }

  // calculate hash with MD5
  HashFunction hashFunction = Hashing.md5();
  Hasher hasher = hashFunction.newHasher();
  boolean hashingSuccess = true;

  // Manage a list
  if (entity instanceof List) {
    List<?> entities = (List) entity;
    for (Object simpleEntity : entities) {
      hashingSuccess = addHash(simpleEntity, hasher);
      if (!hashingSuccess) {
        break;
      }
    }
  } else {
    hashingSuccess = addHash(entity, hasher);
  }

  // if we're able to handle the hash
  if (hashingSuccess) {

    // get result of the hash
    HashCode hashCode = hasher.hash();

    // Create the entity tag
    EntityTag entityTag = new EntityTag(hashCode.toString());

    // Check the etag
    Response.ResponseBuilder builder = request.evaluatePreconditions(entityTag);

    // not modified ?
    if (builder != null) {
      containerResponse.setResponse(builder.tag(entityTag).build());
    } else {
      // it has been changed, so send response with new ETag and entity
      Response.ResponseBuilder responseBuilder =
          Response.fromResponse(containerResponse.getResponse()).tag(entityTag);
      containerResponse.setResponse(responseBuilder.build());
    }
  }
}
 

public String getContentId(){
    HashFunction md5 = Hashing.md5();
    return md5.hashString(contentType + contentName).toString();
}
 

public static String getMD5Checksum(final InputStream is) throws IOException {
    final HashFunction hf = Hashing.md5();

    final HashCode hashCode = hf.hashBytes(getBytesFromInputStream(is));

    return hashCode.toString();
}