Java Code Examples for com.google.common.hash.Hasher#putBytes()

/**
 * Obtain the checksum of a file.
 *
 * @param keyType The type of hash function. e.g. SHA-1, SHA-256.
 * @param path The path to the file.
 * @throws IOException
 */
public static String getChecksum(KeyType keyType, Path path)
    throws IOException, InterruptedException {
  Hasher hasher = keyType.newHasher();
  byte[] byteBuffer = new byte[BUFFER_SIZE];
  try (InputStream stream = path.getInputStream()) {
    int numBytesRead = stream.read(byteBuffer);
    while (numBytesRead != -1) {
      if (numBytesRead != 0) {
        // If more than 0 bytes were read, add them to the hash.
        hasher.putBytes(byteBuffer, 0, numBytesRead);
      }
      if (Thread.interrupted()) {
        throw new InterruptedException();
      }
      numBytesRead = stream.read(byteBuffer);
    }
  }
  return hasher.hash().toString();
}
 

/** Hash the files pointed to by the source paths. */
@VisibleForTesting
static HashCode hashSourcePathGroup(
    FileHashLoader fileHashLoader,
    SourcePathResolverAdapter resolver,
    ImmutableList<SourcePath> paths)
    throws IOException {
  if (paths.size() == 1) {
    return hashSourcePath(paths.get(0), fileHashLoader, resolver);
  }
  Hasher hasher = Hashing.md5().newHasher();
  for (SourcePath path : paths) {
    hasher.putBytes(hashSourcePath(path, fileHashLoader, resolver).asBytes());
  }
  return hasher.hash();
}
 

private void assertCRC32C(int crc, byte... data) throws IOException {
    byte[] buf = new byte[100];
    assertThat(buf.length, greaterThanOrEqualTo(data.length));

    CRC32CInputStream in = new CRC32CInputStream(new ByteArrayInputStream(data), true);
    assertEquals(data.length, in.read(buf));
    int checksum = in.getChecksum();
    in.close();
    assertEquals("Expected " + Util.toHex(crc)
            + ", calculated " + Util.toHex(checksum), crc, checksum);

    // make sure we calculate the same value as the hasher:
    {
        Hasher hasher = Hashing.crc32c().newHasher();
        hasher.putBytes(data);
        assertEquals("Expected " + Util.toHex(crc)
                + ", calculated " + Util.toHex(hasher.hash().asInt()), crc, hasher.hash().asInt());
    }
}
 

@Test
public void testBasics() {
    Hasher hasher = NullHasher.INSTANCE;
    assertEquals(0, hasher.hash().asInt());
    hasher.putBoolean(false);
    hasher.putByte((byte) 3);
    hasher.putBytes(new byte[0]);
    hasher.putBytes(null, 3, 3);
    hasher.putChar('c');
    hasher.putDouble(3.3);
    hasher.putFloat(3.4f);
    hasher.putInt(7);
    hasher.putLong(3);
    hasher.putObject(null, null);
    hasher.putShort((short) 7);
    hasher.putString(null, null);
    hasher.putUnencodedChars(null);
}
 

public Optional<byte[]> calculateSha512() {
    Hasher hasher = sha512().newHasher();
    int size = segments.size();
    if (segments.isEmpty() && contentLength != null && contentLength <= 0) {
        return of(EMPTY_SHA512);
    } else if (size == 1) {
        return segments.first().getReadSha512();
    } else if (size >= 2) {
        for (TransientSegment transientSegment : segments) {
            hasher.putBytes(transientSegment.getReadSha512().get());
        }
        return of(hasher.hash().asBytes());
    } else {
        return absent();
    }
}
 

public Optional<byte[]> calculateMd5() {
    Hasher hasher = md5().newHasher();
    int size = segments.size();
    if (segments.isEmpty() && contentLength != null && contentLength <= 0) {
        return of(EMPTY_MD5);
    } else if (size == 1) {
        return segments.first().getReadMd5();
    } else if (size >= 2) {
        for (TransientSegment transientSegment : segments) {
            hasher.putBytes(transientSegment.getReadMd5().get());
        }
        return of(hasher.hash().asBytes());
    } else {
        return absent();
    }
}
 

private static FileArtifactValue hash(Path path) throws IOException {
  DigestHashFunction hashFn = path.getFileSystem().getDigestFunction();
  Hasher hasher = hashFn.getHashFunction().newHasher();
  int bytesCopied = 0;
  try (InputStream in = path.getInputStream()) {
    byte[] buffer = new byte[1024];
    int len;
    while ((len = in.read(buffer)) > 0) {
      hasher.putBytes(buffer, 0, len);
      bytesCopied += len;
    }
  }
  return FileArtifactValue.createForVirtualActionInput(
      hasher.hash().asBytes(),
      bytesCopied);
}
 

@Override
public int hashCode() {
    Hasher hasher = Hashing.goodFastHash(MIN_BITS).newHasher();
    hasher.putBytes(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
    hasher.putBytes(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
    int qualifierLength = cell.getQualifierLength();
    int qualifierOffset = cell.getQualifierOffset();
    if (isShadowCell()) {
        qualifierLength = qualifierLengthFromShadowCellQualifier(cell.getQualifierArray(),
                cell.getQualifierOffset(),
                cell.getQualifierLength());
        if (startsWith(cell.getQualifierArray(), cell.getQualifierOffset(),
                cell.getQualifierLength(), SHADOW_CELL_PREFIX)) {
            qualifierOffset = qualifierOffset + SHADOW_CELL_PREFIX.length;
        }
    }

    hasher.putBytes(cell.getQualifierArray(),qualifierOffset , qualifierLength);
    hasher.putLong(cell.getTimestamp());
    return hasher.hash().asInt();
}
 

public static ImmutableSet<Path> hashPath(
    Hasher hasher,
    ProjectFileHashLoader fileHashLoader,
    ProjectFilesystem projectFilesystem,
    Path root)
    throws IOException {
  Preconditions.checkArgument(
      !root.equals(EMPTY_PATH), "Path to hash (%s) must not be empty", root);
  ImmutableSet.Builder<Path> children = ImmutableSet.builder();
  for (Path path : ImmutableSortedSet.copyOf(projectFilesystem.getFilesUnderPath(root))) {
    FastPaths.hashPathFast(hasher, path);
    if (!root.equals(path)) {
      children.add(root.relativize(path));
    }
    hasher.putBytes(fileHashLoader.get(path).asBytes());
  }
  return children.build();
}
 

/**
 * The initial hashing phase of a node, which hashes everything except its dependencies.
 *
 * @return the partial {@link Hasher}.
 */
private Hasher startNode(TargetNode<?> node, Object nodeAttributes) {
  Hasher hasher = hashFunction.newHasher();

  // Hash the node's build target and rules.
  LOG.verbose("Hashing node %s", node);
  StringHashing.hashStringAndLength(hasher, node.getBuildTarget().toString());
  JsonObjectHashing.hashJsonObject(hasher, nodeAttributes);
  hasher.putString(ruleKeyConfiguration.getCoreKey(), StandardCharsets.UTF_8);

  // Hash the contents of all input files and directories.
  ProjectFilesystem cellFilesystem = node.getFilesystem();
  for (ForwardRelativePath input : ImmutableSortedSet.copyOf(node.getInputs())) {
    try {
      hasher.putBytes(fileHashLoader.get(cellFilesystem.resolve(input)).asBytes());
    } catch (IOException e) {
      throw new HumanReadableException(
          e, "Error reading path %s for rule %s", input, node.getBuildTarget());
    }
  }

  return hasher;
}
 

private static byte[] download(String path, String hash, IntConsumer progress) throws IOException, VerificationException
{
	HashFunction hashFunction = Hashing.sha256();
	Hasher hasher = hashFunction.newHasher();
	ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();

	URL url = new URL(path);
	HttpURLConnection conn = (HttpURLConnection) url.openConnection();
	conn.setRequestProperty("User-Agent", USER_AGENT);
	conn.getResponseCode();

	InputStream err = conn.getErrorStream();
	if (err != null)
	{
		err.close();
		throw new IOException("Unable to download " + path + " - " + conn.getResponseMessage());
	}

	int downloaded = 0;
	try (InputStream in = conn.getInputStream())
	{
		int i;
		byte[] buffer = new byte[1024 * 1024];
		while ((i = in.read(buffer)) != -1)
		{
			byteArrayOutputStream.write(buffer, 0, i);
			hasher.putBytes(buffer, 0, i);
			downloaded += i;
			progress.accept(downloaded);
		}
	}

	HashCode hashCode = hasher.hash();
	if (!hash.equals(hashCode.toString()))
	{
		throw new VerificationException("Unable to verify resource " + path + " - expected " + hash + " got " + hashCode.toString());
	}

	return byteArrayOutputStream.toByteArray();
}
 

@Override
public byte[] parse(final URI uri, final HttpResponse httpResponse, final LinkReceiver linkReceiver) throws IOException {
	if (hashFunction == null) return null;
	final InputStream is = httpResponse.getEntity().getContent();
	final Hasher hasher = init(crossAuthorityDuplicates? null : uri);
	for(int length; (length = is.read(buffer, 0, buffer.length)) > 0;) hasher.putBytes(buffer, 0, length);
	return hasher.hash().asBytes();
}
 

@Override
public int hashCode() {
    HashFunction hf = Hashing.goodFastHash(32);
    Hasher h = hf.newHasher();
    h.putInt(slots.size());
    for (int i=0; i<slots.size(); i++) {
        h.putInt(slots.get(i).size());
        for (int j=0; j<slots.size(); j++) {
            h.putBytes(slots.get(i).get(j).getLowerRange());
            h.putBytes(slots.get(i).get(j).getUpperRange());
        }
    }
    return h.hash().asInt();
}
 

/**
 * Finish up hashing a node by including its dependencies.
 *
 * @return the nodes {@link HashCode}.
 */
private HashCode finishNode(
    BuildTarget node, Hasher hasher, List<Pair<BuildTarget, HashCode>> depPairs) {
  for (Pair<BuildTarget, HashCode> depPair : depPairs) {
    LOG.verbose(
        "Node %s: adding dependency %s (%s)", node, depPair.getFirst(), depPair.getSecond());
    StringHashing.hashStringAndLength(hasher, depPair.getFirst().toString());
    hasher.putBytes(depPair.getSecond().asBytes());
  }
  return hasher.hash();
}
 

/** Computes a 16-bit checksum of the contents of the specific ByteBuffer and channel name. */
private static int computeChecksum(byte[] buf, int offset, int length, String channel) {
    Hasher hasher = Hashing.murmur3_32().newHasher();
    hasher.putBytes(buf, offset, length);
    hasher.putUnencodedChars(channel);
    return hasher.hash().asInt() & 0xffff;
}
 

@Override
public int hashCode() {
    HashFunction hf = Hashing.goodFastHash(32);
    Hasher h = hf.newHasher();
    h.putInt(slots.size());
    for (int i=0; i<slots.size(); i++) {
        h.putInt(slots.get(i).size());
        for (int j=0; j<slots.size(); j++) {
            h.putBytes(slots.get(i).get(j).getLowerRange());
            h.putBytes(slots.get(i).get(j).getUpperRange());
        }
    }
    return h.hash().asInt();
}
 

public HashCode getOutputHash(FileHashLoader fileHashLoader) throws IOException {
  Hasher hasher = Hashing.md5().newHasher();
  for (Path path : getRecordedPaths()) {
    hasher.putBytes(fileHashLoader.get(projectFilesystem.resolve(path)).asBytes());
  }
  return hasher.hash();
}
 

private static HashCode getCombinedHashForFiles(Set<File> files) {
  List<HashCode> hashesAsBytes =
      files.stream()
          .parallel()
          .map(file -> getFileHash(file))
          .filter(Objects::nonNull)
          .collect(toList());

  // Update the hasher separately because Hasher.putBytes() is not safe for parallel operations
  Hasher hasher = Hashing.sha256().newHasher();
  for (HashCode hash : hashesAsBytes) {
    hasher.putBytes(hash.asBytes());
  }
  return hasher.hash();
}
 

private Optional<SourcePath> getXctool(
    ProjectFilesystem projectFilesystem,
    BuildRuleParams params,
    TargetConfiguration targetConfiguration,
    ActionGraphBuilder graphBuilder) {
  // If xctool is specified as a build target in the buck config, it's wrapping ZIP file which
  // we need to unpack to get at the actual binary.  Otherwise, if it's specified as a path, we
  // can use that directly.
  if (appleConfig.getXctoolZipTarget(targetConfiguration).isPresent()) {
    BuildRule xctoolZipBuildRule =
        graphBuilder.getRule(appleConfig.getXctoolZipTarget(targetConfiguration).get());

    // Since the content is unzipped in a directory that might differ for each cell the tests are
    // from, we append a flavor that depends on the root path of the projectFilesystem
    // in order to get a different rule for each cell the tests are from.
    String relativeRootPathString =
        xctoolZipBuildRule
            .getProjectFilesystem()
            .relativize(projectFilesystem.getRootPath())
            .toString();
    Hasher hasher = Hashing.sha1().newHasher();
    hasher.putBytes(relativeRootPathString.getBytes(Charsets.UTF_8));
    String sha1Hash = hasher.hash().toString();

    BuildTarget unzipXctoolTarget =
        xctoolZipBuildRule
            .getBuildTarget()
            .withAppendedFlavors(UNZIP_XCTOOL_FLAVOR)
            .withAppendedFlavors(InternalFlavor.of(sha1Hash));
    Path outputDirectory =
        BuildTargetPaths.getGenPath(
            xctoolZipBuildRule.getProjectFilesystem(), unzipXctoolTarget, "%s/unzipped");
    graphBuilder.computeIfAbsent(
        unzipXctoolTarget,
        ignored -> {
          BuildRuleParams unzipXctoolParams =
              params
                  .withDeclaredDeps(ImmutableSortedSet.of(xctoolZipBuildRule))
                  .withoutExtraDeps();
          return new AbstractBuildRuleWithDeclaredAndExtraDeps(
              unzipXctoolTarget, xctoolZipBuildRule.getProjectFilesystem(), unzipXctoolParams) {
            @Override
            public ImmutableList<Step> getBuildSteps(
                BuildContext context, BuildableContext buildableContext) {
              buildableContext.recordArtifact(outputDirectory);
              return new ImmutableList.Builder<Step>()
                  .addAll(
                      MakeCleanDirectoryStep.of(
                          BuildCellRelativePath.fromCellRelativePath(
                              context.getBuildCellRootPath(),
                              getProjectFilesystem(),
                              outputDirectory)))
                  .add(
                      new UnzipStep(
                          getProjectFilesystem(),
                          context
                              .getSourcePathResolver()
                              .getAbsolutePath(
                                  Objects.requireNonNull(
                                      xctoolZipBuildRule.getSourcePathToOutput())),
                          outputDirectory,
                          Optional.empty()))
                  .build();
            }

            @Override
            public SourcePath getSourcePathToOutput() {
              return ExplicitBuildTargetSourcePath.of(getBuildTarget(), outputDirectory);
            }
          };
        });
    return Optional.of(
        ExplicitBuildTargetSourcePath.of(
            unzipXctoolTarget, outputDirectory.resolve("bin/xctool")));
  } else if (appleConfig.getXctoolPath().isPresent()) {
    return Optional.of(PathSourcePath.of(projectFilesystem, appleConfig.getXctoolPath().get()));
  } else {
    return Optional.empty();
  }
}
 

/**
 * Return a string representation, as hexadecimal digits, of some hash of the directory.
 *
 * <p>The hash itself is computed according to the design document
 * https://github.com/bazelbuild/proposals/blob/master/designs/2018-07-13-repository-hashing.md
 * and takes enough information into account, to detect the typical non-reproducibility
 * of source-like repository rules, while leaving out what will change from invocation to
 * invocation of a repository rule (in particular file owners) and can reasonably be ignored
 * when considering if a repository is "the same source tree".
 *
 * @return a string representation of the bash of the directory
 * @throws IOException if the digest could not be computed for any reason
 */
public String getDirectoryDigest() throws IOException {
  List<String> entries = new ArrayList<String>(fileSystem.getDirectoryEntries(this));
  Collections.sort(entries);
  Hasher hasher = fileSystem.getDigestFunction().getHashFunction().newHasher();
  for (String entry : entries) {
    Path path = this.getChild(entry);
    FileStatus stat = path.stat(Symlinks.NOFOLLOW);
    hasher.putUnencodedChars(entry);
    if (stat.isFile()) {
      if (path.isExecutable()) {
        hasher.putChar('x');
      } else {
        hasher.putChar('-');
      }
      hasher.putBytes(path.getDigest());
    } else if (stat.isDirectory()) {
      hasher.putChar('d').putUnencodedChars(path.getDirectoryDigest());
    } else if (stat.isSymbolicLink()) {
      PathFragment link = path.readSymbolicLink();
      if (link.isAbsolute()) {
        try {
          Path resolved = path.resolveSymbolicLinks();
          if (resolved.isFile()) {
            if (resolved.isExecutable()) {
              hasher.putChar('x');
            } else {
              hasher.putChar('-');
            }
            hasher.putBytes(resolved.getDigest());
          } else {
            // link to a non-file: include the link itself in the hash
            hasher.putChar('l').putUnencodedChars(link.toString());
          }
        } catch (IOException e) {
          // dangling link: include the link itself in the hash
          hasher.putChar('l').putUnencodedChars(link.toString());
        }
      } else {
        // relative link: include the link itself in the hash
        hasher.putChar('l').putUnencodedChars(link.toString());
      }
    } else {
      // Neither file, nor directory, nor symlink. So do not include further information
      // in the hash, asuming it will not be used during the BUILD anyway.
      hasher.putChar('s');
    }
  }
  return hasher.hash().toString();
}