com.google.common.hash.Hashing Java Examples

boolean isValid()
{
	File file = getJarFile();

	try
	{
		if (file.exists())
		{
			String hash = Files.asByteSource(file).hash(Hashing.sha256()).toString();
			if (this.hash.equals(hash))
			{
				return true;
			}
		}
	}
	catch (IOException e)
	{
	}
	return false;
}
 

public static Path load(Path pluginFolder) throws IOException {
    Path libFolder = pluginFolder.resolve("lib");
    Path path = libFolder.resolve(SNAKE_YAML_JAR);

    if (Files.notExists(path)) {
        Files.createDirectories(libFolder);

        URL url = new URL(SNAKE_YAML);

        String hash;

        try (HashingInputStream his = new HashingInputStream(Hashing.sha1(), url.openStream());
             ReadableByteChannel source = Channels.newChannel(his);
             FileChannel out = FileChannel.open(path, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
            out.transferFrom(source, 0, Long.MAX_VALUE);
            hash = his.hash().toString();
        }

        if (!hash.equals(EXPECTED_HASH)) {
            Files.delete(path);
            throw new IOException("Hash mismatch in " + SNAKE_YAML_JAR + ": expected " + EXPECTED_HASH);
        }
    }

    return path;
}
 

@Override
public QueuePartition getPartition(final FlowFileRecord flowFile, final QueuePartition[] partitions,  final QueuePartition localPartition) {
    final int hash = hash(flowFile);

    // The consistentHash method appears to always return a bucket of '1' if there are 2 possible buckets,
    // so in this case we will just use modulo division to avoid this. I suspect this is a bug with the Guava
    // implementation, but it's not clear at this point.
    final int index;
    if (partitions.length < 3) {
        index = Math.floorMod(hash, partitions.length);
    } else {
        index = Hashing.consistentHash(hash, partitions.length);
    }

    if (logger.isDebugEnabled()) {
        final List<String> partitionDescriptions = new ArrayList<>(partitions.length);
        for (final QueuePartition partition : partitions) {
            partitionDescriptions.add(partition.getSwapPartitionName());
        }

        logger.debug("Assigning Partition {} to {} based on {}", index, flowFile.getAttribute(CoreAttributes.UUID.key()), partitionDescriptions);
    }

    return partitions[index];
}
 

@Before
public void setup() {

    baseCuboidId = (1L << ROW_LENGTH) - 1;
    List<Long> allCuboids = Lists.newArrayList();
    List<Integer[]> allCuboidsBitSetList = Lists.newArrayList();
    for (long i = 1; i < baseCuboidId; i++) {
        allCuboids.add(i);
        addCuboidBitSet(i, allCuboidsBitSetList);
    }

    allCuboidsBitSet = allCuboidsBitSetList.toArray(new Integer[allCuboidsBitSetList.size()][]);
    System.out.println("Totally have " + allCuboidsBitSet.length + " cuboids.");
    allCuboidsHLL = new HLLCounter[allCuboids.size()];
    for (int i = 0; i < allCuboids.size(); i++) {
        allCuboidsHLL[i] = new HLLCounter(14);
    }

    //  hf = Hashing.goodFastHash(32);
    //        hf = Hashing.md5();
    hf = Hashing.murmur3_32();
}
 

/** Hash the files pointed to by the source paths. */
@VisibleForTesting
static HashCode hashSourcePathGroup(
    FileHashLoader fileHashLoader,
    SourcePathResolverAdapter resolver,
    ImmutableList<SourcePath> paths)
    throws IOException {
  if (paths.size() == 1) {
    return hashSourcePath(paths.get(0), fileHashLoader, resolver);
  }
  Hasher hasher = Hashing.md5().newHasher();
  for (SourcePath path : paths) {
    hasher.putBytes(hashSourcePath(path, fileHashLoader, resolver).asBytes());
  }
  return hasher.hash();
}
 

private long generateFingerprint(Collection<URL> extensions) {
    Collection<Integer> hashes = new ArrayList<>();
    for (URL extUrl : extensions) {
        try {
            HashingInputStream hin = new HashingInputStream(
                    Hashing.crc32(), extUrl.openStream());
            //noinspection StatementWithEmptyBody
            while (hin.read() != -1) {
                // Do nothing. Reading all input stream to update hash.
            }
            hashes.add(hin.hash().asInt());
        } catch (IOException e) {
            throw new IllegalArgumentException(e);
        }
    }
    //  FIXME: how to include behaviours in the hash?
    int low = Arrays.hashCode(hashes.toArray());
    int high = pipelineModel.hashCode();
    return ByteBuffer.allocate(8).putInt(high).putInt(low).getLong(0);
}
 

@Test
public void testNioUpload() throws Exception {
    if (log.isInfoEnabled()) log.info("TestCase: " + testCase);

    FileUploadClient fileUploadClient = new FileUploadClient();

    Metadata metadata = new Metadata();
    FileMetadata fileMetadata = new FileMetadata();
    fileMetadata.setFile(testCase.testFile.getAbsolutePath());
    fileMetadata.setSize(testCase.testFile.length());
    fileMetadata.setChecksum(Files.hash(testCase.testFile, Hashing.sha256()).toString());
    metadata.setFilesMetadata(Collections.singletonList(fileMetadata));

    VerificationItems verificationItems = fileUploadClient.uploadFile(testCase.testFile, metadata, testCase.url);

    List<VerificationItem> verificationItemList = verificationItems.getVerificationItems();
    for (VerificationItem verificationItem : verificationItemList){
        Assert.assertEquals("Not matching " + verificationItem, "MATCHING", verificationItem.getStatus());
    }

}
 

@Test
   public void simpleDownloadTest() throws InstantiationException,
           IllegalAccessException, IllegalArgumentException,
		InvocationTargetException, NoSuchFieldException, SecurityException, OneDriveException, IOException, InterruptedException {
	OneDriveSDK api = TestSDKFactory.getInstance();

	OneFolder folder = api.getFolderByPath("/IntegrationTesting/FolderForDownload");
	List<OneFile> files = folder.getChildFiles();


	for (OneFile file : files){
		File localCopy = File.createTempFile(file.getName(), ".bin");

		OneDownloadFile f = file.download(localCopy);
		f.startDownload();

		HashCode code = Files.hash(localCopy, Hashing.sha1());
           assertEquals(file.getName() + " mismatch", code.toString().toUpperCase(), file.getSHA1Hash());
       }
}
 

public static String getNameWithHash(String resourceDirName, String name) {
  String hashname = resources.get(name);
  if (hashname == null) {
    if (resourcePathRoot == null) {
      if (!warningShown) {
        Logger.warn(CachedResourceFileNameHelper.class.getName()
            + " is not initialized by the servlet context or the servlet is not deployed in an exploded way (e.g. when serving directly from war-file). It won't add file-hashes to resource file names.");
        warningShown = true;
      }
      hashname = name;
    } else {
      String nameNoSuffix = name.replaceAll("(\\?.*)$", "");
      String realPath = resourcePathRoot + File.separator + resourceDirName + File.separator + nameNoSuffix;
      try {
        String hashStr = Files.asByteSource(new File(realPath)).hash(Hashing.md5()).toString();
        hashname = nameNoSuffix + "?" + hashStr;
      } catch (IOException e) {
        Logger.error(e);
        hashname = name;
      }
      resources.put(name, hashname);
    }
  }
  return hashname;
}
 

/**
 * Uploads a byte array using FileResource and ExternalFileResource
 *
 * @param name  name of the file to be stored
 * @param bytes the byte array representing the file to be stored
 * @return url pointing to the uploaded resource
 */
private String uploadImage( String name, byte[] bytes )
    throws IOException
{
    FileResource fileResource = new FileResource(
        name,
        MimeTypeUtils.IMAGE_PNG.toString(), // All files uploaded from PushAnalysis is PNG.
        bytes.length,
        ByteSource.wrap( bytes ).hash( Hashing.md5() ).toString(),
        FileResourceDomain.PUSH_ANALYSIS
    );

    String accessToken = saveFileResource( fileResource, bytes );

    return dhisConfigurationProvider.getServerBaseUrl() + "/api/externalFileResources/" + accessToken;

}
 

@Override
public boolean test(String password, String passwordHash) {
  requireNonNull(password, "Required non-null password");
  requireNonNull(passwordHash, "Required non-null password's hash");
  // retrieve salt from the hash
  final int passwordHashLength = ENCRYPTED_PASSWORD_BYTES_LENGTH * 2;
  if (passwordHash.length() < passwordHashLength + SALT_BYTES_LENGTH * 2) {
    return false;
  }
  final HashCode saltHash = HashCode.fromString(passwordHash.substring(passwordHashLength));
  // sha1(password + salt)
  final HashCode hash =
      Hashing.sha512()
          .hashBytes(Bytes.concat(password.getBytes(PWD_CHARSET), saltHash.asBytes()));
  // test sha1(password + salt) + salt == passwordHash
  return (hash.toString() + saltHash.toString()).equals(passwordHash);
}
 

@Test
public void hashDoesNotDependOnFilesystemIterationOrder() throws IOException {
  SettableFakeClock clock = new SettableFakeClock(1000, 0);
  FakeProjectFilesystem filesystem1 = new FakeProjectFilesystem(clock);
  filesystem1.touch(Paths.get("foo/foo.txt"));
  filesystem1.touch(Paths.get("foo/bar.txt"));
  filesystem1.touch(Paths.get("foo/baz.txt"));

  FakeProjectFilesystem filesystem2 = new FakeProjectFilesystem(clock);
  filesystem2.touch(Paths.get("foo/bar.txt"));
  filesystem2.touch(Paths.get("foo/baz.txt"));
  filesystem2.touch(Paths.get("foo/foo.txt"));

  Hasher hasher1 = Hashing.sha1().newHasher();
  PathHashing.hashPath(hasher1, fileHashCache, filesystem1, Paths.get("foo"));

  Hasher hasher2 = Hashing.sha1().newHasher();
  PathHashing.hashPath(hasher2, fileHashCache, filesystem2, Paths.get("foo"));

  assertThat(hasher1.hash(), equalTo(hasher2.hash()));
}
 

@BeforeMethod
public void setup() throws Exception {
    pulsar = mock(PulsarService.class);
    executor = OrderedScheduler.newSchedulerBuilder().numThreads(1).name("test").build();
    zkc = MockZooKeeper.newInstance();
    zkCache = new LocalZooKeeperCache(zkc, 30, executor);
    localCache = new LocalZooKeeperCacheService(zkCache, null);

    // set mock pulsar localzkcache
    LocalZooKeeperCacheService localZkCache = mock(LocalZooKeeperCacheService.class);
    ZooKeeperDataCache<LocalPolicies> poilciesCache = mock(ZooKeeperDataCache.class);
    when(pulsar.getLocalZkCacheService()).thenReturn(localZkCache);
    when(localZkCache.policiesCache()).thenReturn(poilciesCache);
    doNothing().when(poilciesCache).registerListener(any());
    bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());

    doReturn(zkCache).when(pulsar).getLocalZkCache();
    doReturn(localCache).when(pulsar).getLocalZkCacheService();
}
 

private HashCode getHeaderSymlinkTreeHashCode(
    ImmutableSortedMap<Path, Path> contents,
    boolean shouldCreateHeadersSymlinks,
    boolean shouldCreateHeaderMap) {
  Hasher hasher = Hashing.sha1().newHasher();
  hasher.putBytes(ruleKeyConfiguration.getCoreKey().getBytes(Charsets.UTF_8));
  String symlinkState = shouldCreateHeadersSymlinks ? "symlinks-enabled" : "symlinks-disabled";
  byte[] symlinkStateValue = symlinkState.getBytes(Charsets.UTF_8);
  hasher.putInt(symlinkStateValue.length);
  hasher.putBytes(symlinkStateValue);
  String hmapState = shouldCreateHeaderMap ? "hmap-enabled" : "hmap-disabled";
  byte[] hmapStateValue = hmapState.getBytes(Charsets.UTF_8);
  hasher.putInt(hmapStateValue.length);
  hasher.putBytes(hmapStateValue);
  hasher.putInt(0);
  for (Map.Entry<Path, Path> entry : contents.entrySet()) {
    byte[] key = entry.getKey().toString().getBytes(Charsets.UTF_8);
    byte[] value = entry.getValue().toString().getBytes(Charsets.UTF_8);
    hasher.putInt(key.length);
    hasher.putBytes(key);
    hasher.putInt(value.length);
    hasher.putBytes(value);
  }
  return hasher.hash();
}
 

@Test
public void checkUniformDistribution() {
    final long samples = 100000L;
    final int rounds = 5;
    final double percentMarginOfError = 0.5;
    final int numKeys = 100;
    final Random random = new Random();

    for (int round = 0; round < rounds; round++) {
        logger.info(String.format("checkUniformDistribution - round %s: %d samples", round + 1, samples));

        for (final HashFunction hash: new HashFunction[]{md5(), murmur3_128(), sipHash24(), sha256()}) {
            long sum = 0L;
            final long initialTime = System.currentTimeMillis();
            for (Integer counter = 0; counter < samples; counter++) {
                final int chosen = (int) (random.nextFloat() * (numKeys - Float.MIN_VALUE));
                sum += Hashing.consistentHash(hash.hashInt(chosen), (int) numBackends);
            }

            final long finishTime = System.currentTimeMillis();

            final double result = (numBackends * (numBackends - 1) / 2.0) * (samples / numBackends);

            logger.info(String.format("-> checkUniformDistribution (%s): Time spent (ms): %d. NonUniformDistRatio (smaller is better): %.4f%%",
                    hash, finishTime - initialTime, Math.abs(100.0 * (result-sum) / result)));

            final double topLimit = sum * (1.0 + percentMarginOfError);
            final double bottomLimit = sum * (1.0 - percentMarginOfError);

            try {
                assertThat(result).isGreaterThanOrEqualTo(bottomLimit).isLessThanOrEqualTo(topLimit);
            } catch (AssertionError e) {
                logger.error("Error when testing " + hash);
                throw e;
            }
        }
    }
}
 

@Test
public void testHashCodesM3_32_double() {
  int seed = 123;
  Random rand = new Random(seed);
  HashFunction hf = Hashing.murmur3_32(seed);
  for (int i = 0; i < 1000; i++) {
    double val = rand.nextDouble();
    byte[] data = ByteBuffer.allocate(8).putDouble(val).array();
    int hc1 = hf.hashBytes(data).asInt();
    int hc2 = Murmur3.hash32(data, data.length, seed);
    assertEquals(hc1, hc2);
  }
}
 

@ConstructorProperties({"url", "hash"})
public ResourcePackSendPacket(String url, String hash) {
    this.url = url;
    if(hash != null) {
        this.hash = hash.toLowerCase();
    } else {
        this.hash = Hashing.sha1().hashString(url, Charsets.UTF_8).toString().toLowerCase();
    }
}
 

private static boolean needsDownload(Path file, int size, String sha1) {
	try {
		if (!Files.exists(file))
			return true;
		if (Files.size(file) != size)
			return true;
		@SuppressWarnings("deprecation")
		String hash = Hashing.sha1().hashBytes(Files.readAllBytes(file)).toString();
		log.debug("File hash: " + hash + ", comparing to: " + sha1);
		return !hash.equals(sha1);
	} catch (IOException e) {
		log.error("Could not check file integrity: " + file, e);
		return true;
	}
}
 

public static String getWrittenEvaluationCode(WrittenEvaluation writtenEvaluation) {
    StringBuilder code =
            new StringBuilder().append(writtenEvaluation.getInterval()).append(writtenEvaluation.getFullName())
                    .append(writtenEvaluation.getEvaluationType());
    writtenEvaluation.getAssociatedExecutionCoursesSet().stream().forEach(ec -> code.append(getExecutionCourseCode(ec)));
    return Hashing.murmur3_128().hashBytes(code.toString().getBytes(StandardCharsets.UTF_8)).toString();
}
 

@Override
public LongStream keys() throws IOException {
  return lines()
      .map(line -> line.split(" "))
      .filter(array -> array[3].equals("GETVIDEO"))
      .mapToLong(array -> Hashing.murmur3_128().hashUnencodedChars(array[4]).asLong());
}
 

/**
 * There are two tables which store identities in TableAuthIdentityManagerDAO: One table keyed by a hash of the
 * API key, and an index table ID'd by the identity ID which contains the API key hash.  This second table is used
 * to look up API keys by ID.  It should be rare, but it is possible for an API key record tNAo exist
 * without a corresponding ID.  One possible way for this to happen is grandfathered in API keys
 * created before the introduction of IDs.  TableAuthIdentityManagerDAO should rebuild the index
 * when there is a missing or incorrect index record.  This test verifies that works as expected.
 */
@Test
public void testRebuildIdIndex() {
    DataStore dataStore = new InMemoryDataStore(new MetricRegistry());
    Supplier<String> idSupplier = () -> "id0";
    TableAuthIdentityManagerDAO<ApiKey> tableAuthIdentityManagerDAO = new TableAuthIdentityManagerDAO<>(
            ApiKey.class, dataStore, "__auth:keys", "__auth:internal_ids", "app_global:sys",
            idSupplier, Hashing.sha256());

    tableAuthIdentityManagerDAO.createIdentity("testkey",
            new ApiKeyModification()
                    .withOwner("testowner")
                    .addRoles("role1", "role2"));

    // Verify both tables have been written

    String keyTableId = Hashing.sha256().hashUnencodedChars("testkey").toString();

    Map<String, Object> keyMap = dataStore.get("__auth:keys", keyTableId);
    assertFalse(Intrinsic.isDeleted(keyMap));
    assertEquals(keyMap.get("owner"), "testowner");

    Map<String, Object> indexMap = dataStore.get("__auth:internal_ids", "id0");
    assertFalse(Intrinsic.isDeleted(indexMap));
    assertEquals(indexMap.get("hashedId"), keyTableId);

    // Deliberately delete the index map record
    dataStore.update("__auth:internal_ids", "id0", TimeUUIDs.newUUID(), Deltas.delete(),
            new AuditBuilder().setComment("test delete").build());

    Set<String> roles = tableAuthIdentityManagerDAO.getIdentity("id0").getRoles();
    assertEquals(roles, ImmutableSet.of("role1", "role2"));

    // Verify that the index record is re-created
    indexMap = dataStore.get("__auth:internal_ids", "id0");
    assertFalse(Intrinsic.isDeleted(indexMap));
    assertEquals(indexMap.get("hashedId"), keyTableId);
}
 

@Test
public void returnsCorrectHashCodeAndHashFunctionForSha256() {
  FileHash hash =
      FileHash.ofSha256(
          HashCode.fromString(
              "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"));
  Assert.assertEquals(Hashing.sha256(), hash.getHashFunction());
  Assert.assertEquals(
      HashCode.fromString("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
      hash.getHashCode());
}
 

@Override
public Sha1HashCode computeSha1(Path pathRelativeToProjectRootOrJustAbsolute) throws IOException {
  Path fileToHash = getPathForRelativePath(pathRelativeToProjectRootOrJustAbsolute);
  try {
    // Normally, we would just use `Files.hash(fileToHash.toFile(), Hashing.sha1())`, but if
    // fileToHash is backed by Jimfs, its toFile() method throws an UnsupportedOperationException.
    // Creating the input stream via java.nio.file.Files.newInputStream() avoids this issue.
    ByteSource source =
        new ByteSource() {
          @Override
          public InputStream openStream() throws IOException {
            // No need to wrap with BufferedInputStream because ByteSource uses
            // ByteStreams.copy(), which already buffers.
            return Files.newInputStream(fileToHash);
          }
        };
    HashCode hashCode = source.hash(Hashing.sha1());

    return Sha1HashCode.fromHashCode(hashCode);

  } catch (IOException e) {
    String msg =
        String.format("Error computing Sha1 for %s: %s", fileToHash.toString(), e.getMessage());

    throw new IOException(msg, e);
  }
}
 

/** Create a common name based on a hash of the ApplicationId. This should always be less than 64 characters long. */
@SuppressWarnings("UnstableApiUsage")
private static String commonNameHashOf(ApplicationId application, SystemName system) {
    var hashCode = Hashing.sha1().hashString(application.serializedForm(), Charset.defaultCharset());
    var base32encoded = BaseEncoding.base32().omitPadding().lowerCase().encode(hashCode.asBytes());
    return 'v' + base32encoded + Endpoint.dnsSuffix(system);
}
 

public static Optional<String> fileHash(Optional<VirtualFile> vf) {
  if (!vf.isPresent()) {
    return Optional.empty();
  }
  HashFunction hf = Hashing.md5();
  try {
    byte[] bytes = Files.readAllBytes(Paths.get(vf.get().getPath()));
    HashCode hash = hf.newHasher().putBytes(bytes).hash();
    return Optional.of(hash.toString());
  }
  catch (IOException e) {
    e.printStackTrace();
    return Optional.empty();
  }
}
 

@Override
public boolean isDuplicate(Request request, Task task) {
    Set<String> tempLists = urls.computeIfAbsent(task.getUUID(), k -> Sets.newConcurrentHashSet());
    //初始化已采集网站列表缓存
    if (tempLists.add(request.getUrl())) {//先检查当前生命周期是否抓取过,如果当前生命周期未抓取,则进一步检查ES
        GetResponse response = client.prepareGet(INDEX_NAME, TYPE_NAME,
                Hashing.md5().hashString(request.getUrl(), Charset.forName("utf-8")).toString()
        ).get();
        return response.isExists();
    } else {//如果当前生命周期已抓取,直接置为重复
        return true;
    }

}
 

public static final String getHash(String toBeHashed) {
  //noinspection UnstableApiUsage
  return Hashing.murmur3_128().newHasher()
      .putString(toBeHashed, Charset.defaultCharset())
      .hash()
      .toString();
}
 

public String generateSubscriptionKeyFieldsHash(final SubscriptionBase subscription) {
    final Hasher hasher = Hashing.md5()
            .newHasher()
            .putString(subscription.getOwningApplication(), UTF_8)
            .putString(subscription.getConsumerGroup(), UTF_8);
    subscription.getEventTypes()
            .stream()
            .sorted()
            .forEach(et -> hasher.putString(et, UTF_8));
    return hasher.hash().toString();
}
 

public long computeMD5() throws IOException {
  InputSupplier<InputStream> supplier = new InputSupplier<InputStream>() {
    public InputStream getInput() throws IOException {
      return newInputStream();
    }
  };
  return ByteStreams.hash(supplier, Hashing.md5()).asLong();
}
 

public static String genSignature(HttpServletResponseEx responseEx) {
  Hasher hasher = Hashing.sha256().newHasher();
  byte[] bytes = responseEx.getBodyBytes();
  if (bytes != null) {
    hasher.putBytes(bytes, 0, responseEx.getBodyBytesLength());
  }

  return hasher.hash().toString();
}