Python git.Repo.clone_from() Examples

def test_get_files_in_commit():
    finished = False # Authorize PermissionError on cleanup
    try:
        with tempfile.TemporaryDirectory() as temp_dir:
            Repo.clone_from('https://github.com/lmazuel/swagger-to-sdk.git', temp_dir)
            
            files = get_files_in_commit(temp_dir, "b40451e55b26e3db61ea17bd751181cbf91f60c5")

            assert files == [
                'swaggertosdk/generate_package.py',
                'swaggertosdk/python_sdk_tools.py',
                'swaggertosdk/restapi/sdkbot.py'
            ]

            finished = True
    except PermissionError:
        if finished:
            return
        raise 

def clone_repo(self, local_directory_path=None):
        """
        Clone the Git repository to the given file path.
        :param local_directory_path: The file path to clone the repository to. If None, then use the default
        directory path.
        :return: The Git repository.
        """
        if local_directory_path is None:
            local_directory_path = self.local_directory_path
        return Repo.clone_from(self.git_url, local_directory_path)

    # Protected Methods

    # Private Methods

    # Properties 

def test_publish_branch(self, mock_initialized_remote):
        """Method to test creating and listing tags"""
        # Get a clone of the remote repo
        git = mock_initialized_remote[0]

        branches = git.repo.heads
        assert len(branches) == 1
        assert branches[0].name == "master"

        # Create a branch
        git.create_branch("test_branch1")
        # Publish the branch
        git.publish_branch("test_branch1", "origin")

        branches = git.repo.heads
        assert len(branches) == 2
        assert branches[0].name == "master"
        assert branches[1].name == "test_branch1"

        # Create a new clone and make sure you got your branches
        test_working_dir = os.path.join(tempfile.gettempdir(), uuid.uuid4().hex)
        os.makedirs(test_working_dir)
        test_repo = Repo.clone_from(mock_initialized_remote[3], test_working_dir)
        assert len(test_repo.remotes["origin"].fetch()) == 3 

def update(args):
    """Updates local copies of GTFOBins and LOLBAS"""
    toUpdate = genReposToChange(args)
    if toUpdate:
        for repo in toUpdate:
            print("Checking {0} for updates...".format(repo))
            if not os.path.exists(repos[repo]['dir']):
                print("Local copy of {0} not found, ".format(repo) + 
                      "downloading...")
                Repo.clone_from(repos[repo]['url'], repos[repo]['dir'])
                print(green + "Local copy of {0} downloaded".format(repo) + 
                      reset)
            else:
                rep = Repo(repos[repo]['dir'])
                current = rep.head.commit
                rep.remotes.origin.pull()
                if current == rep.head.commit:
                    print(green + "Local copy of {0} is up to ".format(repo) + 
                          "date" + reset)
                else:
                    print(green + "Local copy of {0} ".format(repo) + 
                          "updated" + reset)
    else:
        errorInvalidRepo() 

def check(expanded_dir: str = ".") -> bool:
    """Checks to see if there have been any updates to the Cookiecutter template used
    to generate this project.
    """
    expanded_dir_path = Path(expanded_dir)
    cruft_file = expanded_dir_path / ".cruft.json"
    if not cruft_file.is_file():
        raise NoCruftFound(expanded_dir_path.resolve())

    cruft_state = json.loads(cruft_file.read_text())
    with TemporaryDirectory() as cookiecutter_template_dir:
        repo = Repo.clone_from(cruft_state["template"], cookiecutter_template_dir)
        last_commit = repo.head.object.hexsha
        if last_commit == cruft_state["commit"] or not repo.index.diff(cruft_state["commit"]):
            return True

    return False 

def setUpClass(cls):
        if 'libclang' in autowig.parser:
            autowig.parser.plugin = 'libclang'
        autowig.generator.plugin = 'boost_python_internal'
        cls.srcdir = Path('fp17')
        cls.prefix = Path(Path(sys.prefix).abspath())
        if any(platform.win32_ver()):
            cls.prefix = cls.prefix/'Library'
        if not cls.srcdir.exists():
            Repo.clone_from('https://github.com/StatisKit/FP17.git', cls.srcdir.relpath('.'), recursive=True)
        if any(platform.win32_ver()):
            cls.scons = subprocess.check_output(['where', 'scons.bat']).strip()
        else:
            cls.scons = subprocess.check_output(['which', 'scons']).strip()
        if six.PY3:
            cls.scons = cls.scons.decode("ascii", "ignore")
        subprocess.check_output([cls.scons, 'cpp', '--prefix=' + str(cls.prefix)],
                                 cwd=cls.srcdir)
        cls.incdir = cls.prefix/'include'/'basic' 

def setUpClass(cls):
        if 'libclang' in autowig.parser:
            autowig.parser.plugin = 'libclang'
        cls.srcdir = Path('fp17')
        if not cls.srcdir.exists():
            Repo.clone_from('https://github.com/StatisKit/FP17.git', cls.srcdir.relpath('.'), recursive=True)
        cls.srcdir = cls.srcdir/'share'/'git'/'ClangLite'
        cls.incdir = Path(sys.prefix).abspath()
        if any(platform.win32_ver()):
            cls.incdir = cls.incdir/'Library'
        subprocess.check_output(['scons', 'cpp', '--prefix=' + str(cls.incdir)],
                                cwd=cls.srcdir)
        if any(platform.win32_ver()):
            cls.scons = subprocess.check_output(['where', 'scons.bat']).strip()
        else:
            cls.scons = subprocess.check_output(['which', 'scons']).strip()
        cls.incdir = cls.incdir/'include'/'clanglite' 

def __init__(self, path: str, diff_id: int, comment_file_path: str, token: str, url: str, git_hash: str,
                 push_branch: bool = False):
        self.comment_file_path = comment_file_path
        self.push_branch = push_branch  # type: bool
        self.conduit_token = token  # type: Optional[str]
        self.host = url  # type: Optional[str]
        self._load_arcrc()
        self.diff_id = diff_id  # type: int
        if not self.host.endswith('/api/'):
            self.host += '/api/'
        self.phab = self._create_phab()
        self.base_revision = git_hash  # type: Optional[str]
        self.msg = []  # type: List[str]

        if not os.path.isdir(path):
            print(f'{path} does not exist, cloning repository')
            # TODO: progress of clonning
            self.repo = Repo.clone_from(FORK_REMOTE_URL, path)
        else:
            print('repository exist, will reuse')
            self.repo = Repo(path)  # type: Repo
        os.chdir(path)
        print('working dir', os.getcwd()) 

def __init__(
            self, repo_owner, repo_name, repo_dir=None, do_clone=True,
            from_branch=''):
        self.repo_owner = repo_owner
        self.repo_name = repo_name
        repo_url = 'https://github.com/{0}/{1}'
        self.repo_url = repo_url.format(self.repo_owner, self.repo_name)
        self.repo_dir = repo_dir or self.repo_name
        # by default, start on master.
        self.from_branch = from_branch or 'master'
        self.branch = self.from_branch
        if do_clone:
            self.repo = Repo.clone_from(
                self.repo_url, self.repo_dir, branch=self.from_branch)
        else:
            self.repo = Repo(repo_dir)
        self.git = self.repo.git 

def create_or_update(self, git_url, repo_dir):
        for i in range(1, RETRY + 1):
            try:
                if os.path.isdir(repo_dir):
                     # pull
                     g = git.cmd.Git(repo_dir)
                     g.pull()
                else:
                     # clone
                     Repo.clone_from(git_url, repo_dir)
            except git.GitCommandError as ex:
                logging.info("error:{0}: retry:{1}/{2}".format(ex, i, RETRY))
                time.sleep(10 * i)
                logging.info("retrying")
            else:
                return True
        logging.exception("max retry count reached")
        raise 

def main():
    try:
        shutil.rmtree('./tmp', ignore_errors=True)
        Repo.clone_from(
            'https://github.com/testdrivenio/pycon-sample', './tmp')
    except Exception as e:
        print(e)
        raise
    with stdoutIO() as std_out:
        try:
            p1 = subprocess.Popen(
                ['python3', 'tmp/test.py'],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )
            p2 = p1.stdout.read().decode('utf-8')
            p3 = p1.stderr.read().decode('utf-8')
            if len(p2) > 0:
                print(p2)
            else:
                print(p3)
        except Exception as e:
            print(e)
    print(std_out.getvalue()) 

def download_repo(name, github_repo):
    try:
        directory = os.path.expanduser('~/Repos/%s' % name)
        if os.path.exists(directory):
            shutil.rmtree(directory)

        from git import Repo
        from git import Git

        git_ssh_identity_file = os.path.expanduser('~/.ssh/id_%s' % name)
        git_ssh_cmd = 'ssh -i %s' % git_ssh_identity_file
        Repo.clone_from(
            github_repo, directory, env={
                'GIT_SSH_COMMAND': git_ssh_cmd
            })
        return True, ""
    except Exception as e:
        return False, "download_repo: %s" % e 

def test_configure(self):
        github_token = self.oauth_token
        finished = False
        try:
            with tempfile.TemporaryDirectory() as temp_dir:
                try:
                    Repo.clone_from('https://github.com/lmazuel/TestingRepo.git', temp_dir)
                    repo = Repo(temp_dir)

                    # If it's not throwing, I'm happy enough
                    configure_user(github_token, repo)

                    assert repo.git.config('--get', 'user.name') == 'Laurent Mazuel'
                except Exception as err:
                    print(err)
                    pytest.fail(err)
                else:
                    finished = True
        except PermissionError:
            if finished:
                return
            raise 

def get_raw_data():
    cldr_dates_full_url = "https://github.com/unicode-cldr/cldr-dates-full.git"
    cldr_core_url = "https://github.com/unicode-cldr/cldr-core.git"
    cldr_rbnf_url = "https://github.com/unicode-cldr/cldr-rbnf.git"
    raw_data_directory = "../raw_data"
    cldr_dates_full_dir = "../raw_data/cldr_dates_full/"
    cldr_core_dir = "../raw_data/cldr_core/"
    cldr_rbnf_dir = "../raw_data/cldr_rbnf/"
    if not os.path.isdir(raw_data_directory):
        os.mkdir(raw_data_directory)
        Repo.clone_from(cldr_dates_full_url, cldr_dates_full_dir, branch='master')
        Repo.clone_from(cldr_core_url, cldr_core_dir, branch='master')
        Repo.clone_from(cldr_rbnf_url, cldr_rbnf_dir, branch='master') 

def kiibohd_controller_repo():
    '''
    Downloads a cached copy of the kiibohd controller repo
    '''
    tmp_dir = os.path.join(tempfile.gettempdir(), 'kll_controller_test')
    kll_dir = os.path.join(tmp_dir, 'kll')

    try:
        if not os.path.isdir(tmp_dir):
            # Clone if not available
            Repo.clone_from('https://github.com/kiibohd/controller.git', tmp_dir)
        else:
            # Update otherwise
            repo = Repo(tmp_dir)
            repo.remotes.origin.fetch('+refs/heads/*:refs/remotes/origin/*')
            repo.remotes.origin.pull()
    except exc.GitCommandError:
        # TODO Timeout loop, wait for repo to initialize
        repo = Repo(tmp_dir)
        pass

    try:
        # Check for kll compiler as well (not used during testing, but required for controller tests)
        if not os.path.isdir(kll_dir):
            # Clone if not available
            Repo.clone_from('https://github.com/kiibohd/kll.git', kll_dir)
        else:
            # Update otherwise
            repo_kll = Repo(kll_dir)
            repo_kll.remotes.origin.pull()
    except exc.GitCommandError:
        # TODO Timeout loop, wait for repo to initialize
        repo = Repo(tmp_dir)
        pass

    return tmp_dir 

def _clone_repo(path, git_url):
        """
        Use git to clone locally the neuron in a temp folder
        :return:
        """
        # clone the repo
        logger.debug("[ResourcesManager] GIT clone into folder: %s" % path)
        Utils.print_info("Cloning repository...")
        # if the folder already exist we remove it
        if os.path.exists(path):
            shutil.rmtree(path)
        else:
            os.makedirs(path)
        Repo.clone_from(git_url, path) 

def fetch_git_package(self, config):
        """Make a remote git repository available for local use.

        Args:
            config (dict): git config dictionary

        """
        # only loading git here when needed to avoid load errors on systems
        # without git installed
        from git import Repo

        ref = self.determine_git_ref(config)
        dir_name = self.sanitize_git_path(uri=config['uri'], ref=ref)
        cached_dir_path = os.path.join(self.package_cache_dir, dir_name)

        # We can skip cloning the repo if it's already been cached
        if not os.path.isdir(cached_dir_path):
            logger.debug("Remote repo %s does not appear to have been "
                         "previously downloaded - starting clone to %s",
                         config['uri'],
                         cached_dir_path)
            tmp_dir = tempfile.mkdtemp(prefix='stacker')
            try:
                tmp_repo_path = os.path.join(tmp_dir, dir_name)
                with Repo.clone_from(config['uri'], tmp_repo_path) as repo:
                    repo.head.reference = ref
                    repo.head.reset(index=True, working_tree=True)
                shutil.move(tmp_repo_path, self.package_cache_dir)
            finally:
                shutil.rmtree(tmp_dir)
        else:
            logger.debug("Remote repo %s appears to have been previously "
                         "cloned to %s -- bypassing download",
                         config['uri'],
                         cached_dir_path)

        # Update sys.path & merge in remote configs (if necessary)
        self.update_paths_and_config(config=config,
                                     pkg_dir_name=dir_name) 

def clone_git_repo(repo_path: str, url: str) -> str:
    return GitRepo.clone_from(url=url, to_path=repo_path) 

def clone_repository(repository):
        repo_url = repository.url
        repo_path = repository.get_local_repo_path()
        Frontend.set_auth(repository)

        try:
            repo = Repo.clone_from(repository.url, repository.get_local_repo_path(), branch='master')
            repository.conn_ok = True
        except Exception:
            repository.conn_ok = False

        repository.save() 

def main():
    try:
        import volatility
    except ImportError:
        volpath = os.path.join(VENDOR_ROOT, "volatility")
        setup_script = os.path.join(volpath, "setup.py")

        rmtree(volpath, True)
        Repo.clone_from("https://github.com/volatilityfoundation/volatility.git", volpath)

        os.chdir(volpath)
        call(['python', setup_script, 'install']) 

def main():
    decoders_path = os.path.join(VENDOR_ROOT, 'RATDecoders')

    if os.path.exists(decoders_path):
        repo = Repo(decoders_path)
        repo.remotes.origin.pull()
    else:
        Repo.clone_from("https://github.com/kevthehermit/RATDecoders.git", decoders_path) 

def clone_to_path(https_authenticated_url, folder, branch_or_commit=None):
    """Clone the given URL to the folder.

    :param str branch_or_commit: If specified, switch to this branch. Branch must exist.
    """
    _LOGGER.info("Cloning repo")
    repo = Repo.clone_from(https_authenticated_url, str(folder))
    # Do NOT clone and set branch at the same time, since we allow branch to be a SHA1
    # And you can't clone a SHA1
    if branch_or_commit:
        _LOGGER.info("Checkout branch_or_commit %s", branch_or_commit)
        repo.git.checkout(branch_or_commit)

    _LOGGER.info("Clone success") 

def update(self):
        if self.need_update():
            print("[\033[92mINFO\033[0m] Found an updated version! cloning...")

            Repo.clone_from(self.url, self.tmp_dir)
            os.chdir(self.tmp_dir)
            print("[\033[92mINFO\033[0m] Installing...")
            if self.install(self.new_version):
                os.chdir("..")
                shutil.rmtree(self.tmp_dir)
                return True
        else:
            print("[\033[92mINFO\033[0m] You've got already the last version :)")
        shutil.rmtree(self.tmp_dir)
        return False 

def download_generic_alerts(repo_url, save_location, deploy_key):
    git_obj = cmd.Git(save_location)
    git_ssh_cmd = 'ssh -i %s' % deploy_key

    git_obj.update_environment(GIT_SSH_COMMAND=git_ssh_cmd)

    if not os.path.isdir(save_location):
        logger.debug("Cloning " + str(repo_url) + " into " + str(save_location))
        Repo.clone_from(repo_url, save_location, env={'GIT_SSH_COMMAND': git_ssh_cmd})
    else:
        logger.debug("Updating " + str(save_location))
        git_obj.pull() 

def get_django_repo_object(branch: str) -> Repo:
    if not DJANGO_SOURCE_DIRECTORY.exists():
        DJANGO_SOURCE_DIRECTORY.mkdir(exist_ok=True, parents=False)
        return Repo.clone_from('https://github.com/django/django.git', DJANGO_SOURCE_DIRECTORY,
                               progress=ProgressPrinter(),
                               branch=branch,
                               depth=100)
    else:
        repo = Repo(DJANGO_SOURCE_DIRECTORY)
        return repo 

def clone_git_repo(git_url: str) -> str:
    project_path = tempfile.mkdtemp()
    Repo.clone_from(git_url, project_path)
    return project_path 

def git_clone(repo_url, branch='master', path=''):
    """clones repo to a temporary dir, the path of which is determined by the platform"""

    _tmp_dir = tempfile.mkdtemp(prefix='pyhelm-')
    repo = Repo.clone_from(repo_url, _tmp_dir, branch=branch)

    return os.path.join(_tmp_dir, path) 

def clone_feedstock(feedstock_gh_repo, feedstocks_dir):
    repo = feedstock_gh_repo

    clone_directory = os.path.join(feedstocks_dir, repo.name)
    if not os.path.exists(clone_directory):
        print("Cloning {}".format(repo.name))
        clone = Repo.clone_from(repo.clone_url, clone_directory)
        clone.delete_remote("origin")
    clone = Repo(clone_directory)
    if "upstream" in [remote.name for remote in clone.remotes]:
        clone.delete_remote("upstream")
    clone.create_remote("upstream", url=repo.clone_url) 

def main():
    try:
        payload = json.loads(get_stdin())
        namespace = payload['repository']['owner']['login']
        repo_name = payload['repository']['name']
        url = f'https://github.com/{namespace}/{repo_name}'
    except Exception as e:
        print(ERROR_MSG)
        raise
    try:
        shutil.rmtree(f'./tmp/{repo_name}', ignore_errors=True)
        Repo.clone_from(url, f'./tmp/{repo_name}')
    except Exception as e:
        print(e)
        raise
    with stdoutIO() as std_out:
        try:
            p1 = subprocess.Popen(
                ['python3', f'tmp/{repo_name}/test.py'],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )
            p2 = p1.stdout.read().decode('utf-8')
            p3 = p1.stderr.read().decode('utf-8')
            status = False
            if len(p2) > 0:
                print(p2)
            else:
                print(p3)
                if 'failures' not in p3:
                    status = True
            payload = {
                'repo_name': repo_name,
                'status': status

            }
            r = requests.put(f'{WEB_URL}/projects/update', json=payload)
            print(json.dumps(r.json()))
        except Exception as e:
            print(e) 

def main():
    try:
        payload = json.loads(get_stdin())
        namespace = payload['namespace']
        repo_name = payload['repo_name']
        url = f'https://github.com/{namespace}/{repo_name}'
    except Exception as e:
        print(ERROR_MSG)
        raise
    try:
        shutil.rmtree(f'./tmp/{repo_name}', ignore_errors=True)
        Repo.clone_from(url, f'./tmp/{repo_name}')
    except Exception as e:
        print(e)
        raise
    with stdoutIO() as std_out:
        try:
            p1 = subprocess.Popen(
                ['python3', f'tmp/{repo_name}/test.py'],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )
            p2 = p1.stdout.read().decode('utf-8')
            p3 = p1.stderr.read().decode('utf-8')
            status = False
            if len(p2) > 0:
                print(p2)
            else:
                print(p3)
                if 'failures' not in p3:
                    status = True
        except Exception as e:
            print(e)
    response_object = {
        'status': status,
        'data': std_out.getvalue()
    }
    print(json.dumps(response_object))