org.eclipse.jgit.lib.ObjectLoader Java Examples

private Map<String, BlobWrapper> getContentMapByTreeAndFilter(
        Git git, AbstractTreeIterator tree, TreeFilter filter) throws Exception {
    Map<String, BlobWrapper> contentMap = new LinkedHashMap<>();
    try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) {
        treeWalk.addTree(tree);
        treeWalk.setRecursive(true);
        treeWalk.setFilter(filter);
        while (treeWalk.next()) {
            ObjectId objectId = treeWalk.getObjectId(0);
            ObjectLoader loader = git.getRepository().open(objectId);
            BlobWrapper blobWrapper = BlobWrapper.builder()
                    .blobId(objectId)
                    .content(loader.getBytes())
                    .build();
            contentMap.put(treeWalk.getPathString(), blobWrapper);
        }
    }
    return contentMap;
}
 

/**
 * Method to read a file from a specific revision.
 *
 * @param tree the revision tree that contains the file.
 * @param path the path that leads to the file in the tree.
 * @return a list containing all lines in the file.
 */
public List<String> getFileLines(RevTree tree, String path) throws IOException, GitAPIException {

  try (TreeWalk walk = new TreeWalk(this.repo)) {
    walk.addTree(tree);
    walk.setRecursive(true);
    walk.setFilter(PathFilter.create(path));

    walk.next();
    ObjectId oId = walk.getObjectId(0);

    if (oId == ObjectId.zeroId()) {
      return new LinkedList<>();
    }

    ObjectLoader loader = this.repo.open(oId);

    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    loader.copyTo(stream);

    return IOUtils.readLines(new ByteArrayInputStream(stream.toByteArray()), "UTF-8");
  } catch (Exception e) {
    return new LinkedList<>();
  }
}
 

/**
 * This test case expects one git repository to be present in local file system.
 * Currently this test uses the Baeldung repository i.e. the current checkout repository.
 * It finds the repository by tracking back and scan file system to find .git folder in
 * the file system.
 * 
 * Before running the test case ensure that the .git folder is present.
 * 
 * @throws IOException
 */
@Test
public void testRevWalkDisposeClosesReader() throws IOException {
    try (Repository repo = Helper.openJGitRepository()) {
        try (ObjectReader reader = repo.newObjectReader()) {
            try (RevWalk walk = new RevWalk(reader)) {
                walk.dispose();

                Ref head = repo.exactRef("refs/heads/master");
                System.out.println("Found head: " + head);

                ObjectLoader loader = reader.open(head.getObjectId());
                assertNotNull(loader);
            }
        }
    }
}
 

@Override
public long getContentSize(final String site, final String path) {
    Repository repo = helper.getRepository(site, StringUtils.isEmpty(site) ? GLOBAL : SANDBOX);
    try {
        RevTree tree = helper.getTreeForLastCommit(repo);
        try (TreeWalk tw = TreeWalk.forPath(repo, helper.getGitPath(path), tree)) {
            if (tw != null && tw.getObjectId(0) != null) {
                ObjectId id = tw.getObjectId(0);
                ObjectLoader objectLoader = repo.open(id);
                return objectLoader.getSize();
            }
        }
    } catch (IOException e) {
        logger.error("Error while getting content for file at site: " + site + " path: " + path, e);
    }
    return -1L;
}
 

@NotNull
@Override
public InputStream inputStream(@NotNull GitObject<? extends ObjectId> objectId) throws IOException {
  final ObjectLoader loader = objectId.openObject();
  try (ObjectStream stream = loader.openStream()) {
    final byte[] header = new byte[Constants.POINTER_MAX_SIZE];
    int length = IOUtils.read(stream, header, 0, header.length);
    if (length < header.length) {
      final Meta meta = parseMeta(header, length);
      if (meta != null)
        return getReader(meta).openStream();
    }

    // We need to re-open stream
    return loader.openStream();
  }
}
 

@NotNull
@Override
public String getMd5(@NotNull GitObject<? extends ObjectId> objectId) throws IOException {
  final ObjectLoader loader = objectId.openObject();

  try (ObjectStream stream = loader.openStream()) {
    final Meta meta = parseMeta(stream);
    if (meta != null) {
      final String md5 = getReader(meta).getMd5();
      if (md5 != null)
        return md5;
    }
  }

  return GitFilterHelper.getMd5(this, cacheMd5, null, objectId);
}
 

private void obtainDiff(Repository repository, RevCommit commit, List<String> paths) throws IOException, GitAPIException {
        // and using commit's tree find the path
        RevTree tree = commit.getTree();
        System.out.println("Having tree: " + tree);

        // now try to find a specific file
        TreeWalk treeWalk = new TreeWalk(repository);
        treeWalk.addTree(tree);
        treeWalk.setRecursive(true);
        for (String path : paths) {
            String filePath = SIVA_COMMITS_DIR + commit.getName() + "/" + path;
            File file = new File(filePath);
            if (!file.exists()) {
                treeWalk.setFilter(PathFilter.create(path));
                if (!treeWalk.next()) {
                    throw new IllegalStateException("Did not find expected file '" + path + "'");
                }

                ObjectId objectId = treeWalk.getObjectId(0);
                ObjectLoader loader = repository.open(objectId);
                // and then one can the loader to read the file
//                loader.copyTo(System.out);
                loader.copyTo(FileUtils.openOutputStream(file));
            }
        }
    }
 

/** {@inheritDoc} */
@Override
public void appendNote(String note, String namespace) throws GitException {
    try (Repository repo = getRepository()) {
        ObjectId head = repo.resolve(HEAD); // commit to put a note on

        ShowNoteCommand cmd = git(repo).notesShow();
        cmd.setNotesRef(qualifyNotesNamespace(namespace));
        try (ObjectReader or = repo.newObjectReader();
             RevWalk walk = new RevWalk(or)) {
            cmd.setObjectId(walk.parseAny(head));
            Note n = cmd.call();

            if (n==null) {
                addNote(note,namespace);
            } else {
                ObjectLoader ol = or.open(n.getData());
                StringWriter sw = new StringWriter();
                IOUtils.copy(new InputStreamReader(ol.openStream(), UTF_8),sw);
                sw.write("\n");
                addNote(sw.toString() + normalizeNote(note), namespace);
            }
        }
    } catch (GitAPIException | IOException e) {
        throw new GitException(e);
    }
}
 

private Map<String, RawFile> walkGitObjectTree(Optional<Long> maxFileSize)
        throws IOException,
                SizeLimitExceededException,
                InvalidGitRepository {
    Map<String, RawFile> fileContentsTable = new HashMap<>();
    if (treeWalk == null) {
        return fileContentsTable;
    }
    while (treeWalk.next()) {
        String path = treeWalk.getPathString();
        ObjectId objectId = treeWalk.getObjectId(0);
        if (!repository.hasObject(objectId)) {
            throw new InvalidGitRepository();
        }
        ObjectLoader obj = repository.open(objectId);
        long size = obj.getSize();
        if (maxFileSize.isPresent() && size > maxFileSize.get()) {
            throw new SizeLimitExceededException(
                    Optional.ofNullable(path), size, maxFileSize.get());
        }
        try (ByteArrayOutputStream o = new ByteArrayOutputStream(
                CastUtil.assumeInt(size))) {
            obj.copyTo(o);
            fileContentsTable.put(
                    path, new RepositoryFile(path, o.toByteArray()));
        };
    }
    return fileContentsTable;
}
 

@Override
public long getSize(@NotNull GitObject<? extends ObjectId> objectId) throws IOException {
  final ObjectLoader loader = objectId.openObject();

  try (ObjectStream stream = loader.openStream()) {
    final Meta meta = parseMeta(stream);
    if (meta != null)
      return meta.getSize();
  }

  return loader.getSize();
}
 

private void populateFileContents(Repository repository, RevCommit commit,
		List<String> filePaths, Map<String, String> fileContents, Set<String> repositoryDirectories) throws Exception {
	logger.info("Processing {} {} ...", repository.getDirectory().getParent().toString(), commit.getName());
	RevTree parentTree = commit.getTree();
	try (TreeWalk treeWalk = new TreeWalk(repository)) {
		treeWalk.addTree(parentTree);
		treeWalk.setRecursive(true);
		while (treeWalk.next()) {
			String pathString = treeWalk.getPathString();
			if(filePaths.contains(pathString)) {
				ObjectId objectId = treeWalk.getObjectId(0);
				ObjectLoader loader = repository.open(objectId);
				StringWriter writer = new StringWriter();
				IOUtils.copy(loader.openStream(), writer);
				fileContents.put(pathString, writer.toString());
			}
			if(pathString.endsWith(".java") && pathString.contains("/")) {
				String directory = pathString.substring(0, pathString.lastIndexOf("/"));
				repositoryDirectories.add(directory);
				//include sub-directories
				String subDirectory = new String(directory);
				while(subDirectory.contains("/")) {
					subDirectory = subDirectory.substring(0, subDirectory.lastIndexOf("/"));
					repositoryDirectories.add(subDirectory);
				}
			}
		}
	}
}
 

/**
 * Utility method that converts a note to a string (assuming it's UTF-8).
 */
private String noteToString(Repository repo, Note note)
    throws MissingObjectException, IOException, UnsupportedEncodingException {
  ObjectLoader loader = repo.open(note.getData());
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  loader.copyTo(baos);
  return new String(baos.toByteArray(), "UTF-8");
}
 

private void logObjectContent(ObjectLoader objectLoader, String virtualName)
    throws IOException {
  if (!log.isDebugEnabled()) {
    return;
  }

  try (InputStream input = objectLoader.openStream();
      OutputStream output = TemporaryFile.create(log, virtualName).newOutputStream()) {
    IOUtils.copy(input, output);
  }
}
 

public InputStream getInputStream(BlobIdent ident) {
	try (RevWalk revWalk = new RevWalk(getRepository())) {
		ObjectId commitId = getObjectId(ident.revision, true);
		RevTree revTree = revWalk.parseCommit(commitId).getTree();
		TreeWalk treeWalk = TreeWalk.forPath(getRepository(), ident.path, revTree);
		if (treeWalk != null) {
			ObjectLoader objectLoader = treeWalk.getObjectReader().open(treeWalk.getObjectId(0));
			return objectLoader.openStream();
		} else {
			throw new ObjectNotFoundException("Unable to find blob path '" + ident.path + "' in revision '" + ident.revision + "'");
		}
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

private RawText getRawText(ObjectId id,
		Attributes attributes)
		throws IOException, BinaryBlobException {
	if (id.equals(ObjectId.zeroId()))
		return new RawText(new byte[] {});

	ObjectLoader loader = LfsFactory.getInstance().applySmudgeFilter(
			getRepository(), reader.open(id, OBJ_BLOB),
			attributes.get(Constants.ATTR_MERGE));
	int threshold = PackConfig.DEFAULT_BIG_FILE_THRESHOLD;
	return RawText.load(loader, threshold);
}
 

byte[] getCachedBytes(RevObject obj, ObjectLoader ldr)
		throws LargeObjectException, MissingObjectException, IOException {
	try {
		return ldr.getCachedBytes(5 * MB);
	} catch (LargeObjectException tooBig) {
		tooBig.setObjectId(obj);
		throw tooBig;
	}
}
 

private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
		throws LargeObjectException, CorruptObjectException,
		MissingObjectException, IOException {
	RevObject r;
	int type = ldr.getType();
	switch (type) {
	case Constants.OBJ_COMMIT: {
		final RevCommit c = createCommit(id);
		c.parseCanonical(this, getCachedBytes(c, ldr));
		r = c;
		break;
	}
	case Constants.OBJ_TREE: {
		r = new RevTree(id);
		r.flags |= PARSED;
		break;
	}
	case Constants.OBJ_BLOB: {
		r = new RevBlob(id);
		r.flags |= PARSED;
		break;
	}
	case Constants.OBJ_TAG: {
		final RevTag t = new RevTag(id);
		t.parseCanonical(this, getCachedBytes(t, ldr));
		r = t;
		break;
	}
	default:
		throw new IllegalArgumentException(MessageFormat.format(
				JGitText.get().badObjectType, Integer.valueOf(type)));
	}
	objects.add(r);
	return r;
}
 

/**
 * Extract the RawText object from a id.
 *
 * @param id the id on the object, which in this case is a commit.
 * @return either null or a RawText object.
 */
private RawText toRaw(AbbreviatedObjectId id) {
  try {
    ObjectLoader loader = this.repo.open(id.toObjectId());
    return RawText.load(loader, PackConfig.DEFAULT_BIG_FILE_THRESHOLD);
  } catch (Exception e) {
    return null;
  }
}
 

@Override
public ObjectLoader open(AnyObjectId objectId, int typeHint)
    throws MissingObjectException, IncorrectObjectTypeException, IOException {
  return new AutoCRLFObjectLoader(delegate.open(objectId, typeHint), eolStreamType);
}
 

private void doFormat(DirCacheEntry dirCacheEntry, CodeFormatter formatter) {
  LineRanges lineRanges = computeLineRanges(dirCacheEntry);
  if (lineRanges.isAll()) {
    log.info("Formatting '" + dirCacheEntry.getPathString() + "'");
  } else {
    log.info("Formatting lines " + lineRanges + " of '" + dirCacheEntry.getPathString() + "'");
  }

  try (TemporaryFile temporaryFormattedFile =
      TemporaryFile.create(log, dirCacheEntry.getPathString() + ".formatted")) {
    ObjectId unformattedObjectId = dirCacheEntry.getObjectId();
    log.debug("Unformatted object id is '" + unformattedObjectId + "'");
    ObjectDatabase objectDatabase = repository.getObjectDatabase();
    ObjectLoader objectLoader = objectDatabase.open(unformattedObjectId);

    logObjectContent(objectLoader, dirCacheEntry.getPathString() + ".unformatted");

    try (InputStream content = objectLoader.openStream();
        OutputStream formattedContent = temporaryFormattedFile.newOutputStream()) {
      formatter.format(content, lineRanges, formattedContent);
    }

    long formattedSize = temporaryFormattedFile.size();
    ObjectId formattedObjectId;
    try (InputStream formattedContent = temporaryFormattedFile.newInputStream();
        ObjectInserter objectInserter = objectDatabase.newInserter()) {
      formattedObjectId = objectInserter.insert(OBJ_BLOB, formattedSize, formattedContent);
      objectInserter.flush();
    }

    log.debug("Formatted size is " + formattedSize);
    dirCacheEntry.setLength(formattedSize);
    log.debug("Formatted object id is '" + formattedObjectId + "'");
    dirCacheEntry.setObjectId(formattedObjectId);
  } catch (IOException e) {
    throw new MavenGitCodeFormatException(e);
  }

  if (lineRanges.isAll()) {
    log.info("Formatted '" + dirCacheEntry.getPathString() + "'");
  } else {
    log.info("Formatted lines " + lineRanges + " of '" + dirCacheEntry.getPathString() + "'");
  }
}
 

public AutoCRLFObjectLoader(ObjectLoader delegate, EolStreamType eolStreamType) {
  this.delegate = requireNonNull(delegate);
  this.eolStreamType = requireNonNull(eolStreamType);
}
 

public void testLineEndingsWindows () throws Exception {
    if (!isWindows()) {
        return;
    }
    // lets turn autocrlf on
    StoredConfig cfg = repository.getConfig();
    cfg.setString(ConfigConstants.CONFIG_CORE_SECTION, null, ConfigConstants.CONFIG_KEY_AUTOCRLF, "true");
    cfg.save();
    
    File f = new File(workDir, "f");
    write(f, "a\r\nb\r\n");
    File[] roots = new File[] { f };
    
    GitClient client = getClient(workDir);
    runExternally(workDir, Arrays.asList("git.cmd", "add", "f"));
    DirCacheEntry e1 = repository.readDirCache().getEntry("f");
    client.add(roots, NULL_PROGRESS_MONITOR);
    DirCacheEntry e2 = repository.readDirCache().getEntry("f");
    assertStatus(client.getStatus(roots, NULL_PROGRESS_MONITOR),
            workDir, f, true, Status.STATUS_ADDED, Status.STATUS_NORMAL, Status.STATUS_ADDED, false);
    List<String> res = runExternally(workDir, Arrays.asList("git.cmd", "status", "-s"));
    assertEquals(Arrays.asList("A  f"), res);
    assertEquals(e1.getFileMode(), e2.getFileMode());
    assertEquals(e1.getPathString(), e2.getPathString());
    assertEquals(e1.getRawMode(), e2.getRawMode());
    assertEquals(e1.getStage(), e2.getStage());
    assertEquals(e1.getLength(), e2.getLength());
    assertEquals(e1.getObjectId(), e2.getObjectId());

    write(f, "a\nb\n");
    res = runExternally(workDir, Arrays.asList("git.cmd", "status", "-s"));
    assertEquals(Arrays.asList("AM f"), res);
    assertStatus(client.getStatus(roots, NULL_PROGRESS_MONITOR),
            workDir, f, true, Status.STATUS_ADDED, Status.STATUS_MODIFIED, Status.STATUS_ADDED, false);
    
    res = runExternally(workDir, Arrays.asList("git.cmd", "commit", "-m", "gugu"));
    res = runExternally(workDir, Arrays.asList("git.cmd", "checkout", "--", "f"));
    
    RevCommit commit = Utils.findCommit(repository, "HEAD");
    TreeWalk walk = new TreeWalk(repository);
    walk.reset();
    walk.addTree(commit.getTree());
    walk.setFilter(PathFilter.create("f"));
    walk.setRecursive(true);
    walk.next();
    assertEquals("f", walk.getPathString());
    ObjectLoader loader = repository.getObjectDatabase().open(walk.getObjectId(0));
    assertEquals(4, loader.getSize());
    assertEquals("a\nb\n", new String(loader.getBytes()));
    assertEquals(e1.getObjectId(), walk.getObjectId(0));
    
    res = runExternally(workDir, Arrays.asList("git.cmd", "status", "-s"));
    assertEquals(0, res.size());
    assertStatus(client.getStatus(roots, NULL_PROGRESS_MONITOR),
            workDir, f, true, Status.STATUS_NORMAL, Status.STATUS_NORMAL, Status.STATUS_NORMAL, false);
}
 

@NotNull
public ObjectLoader openObject() throws IOException {
  return repo.newObjectReader().open(object);
}
 

@NotNull
@Override
public InputStream inputStream(@NotNull GitObject<? extends ObjectId> objectId) throws IOException {
  final ObjectLoader loader = objectId.openObject();
  return loader.openStream();
}
 

private void indexBlob(IndexWriter writer, Repository repository, 
		SymbolExtractor<Symbol> extractor, ObjectId blobId, String blobPath) throws IOException {
	Document document = new Document();
	
	document.add(new StoredField(BLOB_INDEX_VERSION.name(), getIndexVersion(extractor)));
	document.add(new StringField(BLOB_HASH.name(), blobId.name(), Store.NO));
	document.add(new StringField(BLOB_PATH.name(), blobPath, Store.NO));
	document.add(new BinaryDocValuesField(BLOB_PATH.name(), new BytesRef(blobPath.getBytes(StandardCharsets.UTF_8))));
	
	String blobName = blobPath;
	if (blobPath.indexOf('/') != -1) 
		blobName = StringUtils.substringAfterLast(blobPath, "/");
	
	document.add(new StringField(BLOB_NAME.name(), blobName.toLowerCase(), Store.NO));
	
	ObjectLoader objectLoader = repository.open(blobId);
	if (objectLoader.getSize() <= MAX_INDEXABLE_SIZE) {
		byte[] bytes = objectLoader.getCachedBytes();
		String content = ContentDetector.convertToText(bytes, blobName);
		if (content != null) {
			document.add(new TextField(BLOB_TEXT.name(), content, Store.NO));
			
			if (extractor != null) {
				List<Symbol> symbols = null;
				try {
					symbols = extractor.extract(blobName, StringUtils.removeBOM(content));
				} catch (Exception e) {
					logger.trace("Can not extract symbols from blob (hash:" + blobId.name() + ", path:" + blobPath + ")", e);
				}
				if (symbols != null) {
					for (Symbol symbol: symbols) {
						String fieldValue = symbol.getName();
						if (fieldValue != null && symbol.isSearchable()) {
							fieldValue = fieldValue.toLowerCase();

							String fieldName;
							if (symbol.isPrimary())
								fieldName = BLOB_PRIMARY_SYMBOLS.name();
							else
								fieldName = BLOB_SECONDARY_SYMBOLS.name();
							document.add(new StringField(fieldName, fieldValue, Store.NO));
						}
					}
					byte[] bytesOfSymbols = SerializationUtils.serialize((Serializable) symbols);
					document.add(new StoredField(BLOB_SYMBOL_LIST.name(), bytesOfSymbols));
				}
			} 
		} else {
			logger.debug("Ignore content of binary file '{}'.", blobPath);
		}
	} else {
		logger.debug("Ignore content of large file '{}'.", blobPath);
	}

	writer.addDocument(document);
}
 

@Override
@Nullable
public <C> C getFileContents(
        final Class<C> c,
        final String[] path,
        @Nullable final C defaultValue,
        final String revision
) throws StoreException.ReadException, JsonProcessingException {
    try {
        if (!ObjectId.isId(revision)) {
            throw new StoreException.ReadException("Malformed id " + revision);
        }
        final ObjectId blobOrCommitId = ObjectId.fromString(revision);

        final ObjectLoader loader = git.getRepository().open(blobOrCommitId);

        if (loader.getType() == Constants.OBJ_COMMIT) {
            // look up the file at this revision
            final RevCommit commit = RevCommit.parse(loader.getCachedBytes());

            final TreeWalk treeWalk2 = new TreeWalk(git.getRepository());
            treeWalk2.addTree(commit.getTree());
            treeWalk2.setRecursive(true);
            final String joinedPath = String.join("/", path);
            treeWalk2.setFilter(PathFilter.create(joinedPath));

            if (!treeWalk2.next()) {
                // it did not find expected file `joinPath` so return default value
                return defaultValue;
            }
            final ObjectId blobId = treeWalk2.getObjectId(0);
            return getFileContents(c, blobId);
        } else if (loader.getType() == Constants.OBJ_BLOB) {
            return getFileContents(c, blobOrCommitId);
        } else {
            throw new StoreException.ReadException("Invalid Object Type " + loader.getType() + " for id " + revision);
        }
    } catch (final IOException e) {
        throw new StoreException.ReadException(e);
    }
}
 

private <C> C getFileContents(final Class<C> c,
        final ObjectId blobId) throws IOException {
    final ObjectLoader loader = git.getRepository().open(blobId);
    final ObjectMapper mapper = Serializers.lenient();
    return mapper.readValue(loader.getBytes(), c);
}
 

private VersionedFile readToByteArray(TreeWalk treeWalk) throws IOException {
	ObjectId objectId = treeWalk.getObjectId(0);
	ObjectLoader loader = repository.open(objectId);
	return new ByteArrayVersionedFile(IOUtils.toByteArray(loader.openStream()));
}