Python pathlib.Path.exists() Examples

def test_logging_failing_explore_sql(tmpdir, sql_error):
    sql_error.metadata["dimension"] = None
    expected_directory = Path(tmpdir) / "queries"
    expected_directory.mkdir(exist_ok=True)

    log_sql_error(
        sql_error.model,
        sql_error.explore,
        sql_error.metadata["sql"],
        tmpdir,
        sql_error.metadata["dimension"],
    )
    expected_path = expected_directory / "eye_exam__users.sql"

    assert Path.exists(expected_path)
    with expected_path.open("r") as file:
        content = file.read()
    assert content == "SELECT age FROM users WHERE 1=2 LIMIT 1" 

def validate_parameter_value(input_name, input_value, min_value=None, max_value=None, possible_values=None):
    if isinstance(input_value, PurePath):
        if not Path.exists(input_value):  # pragma: no cover
            raise ValueError("Given path: " + str(input_value) + " does not exist.")
    if input_value is not None:
        if min_value is not None:
            if input_value < min_value:  # pragma: no cover
                raise ValueError(
                    "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Provide a value greater than or equal to " + str(min_value) + ".")
    if input_value is not None:
        if max_value is not None:
            if input_value > max_value:  # pragma: no cover
                raise ValueError(
                    "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Provide a value less than or equal to " + str(max_value) + ".")

    if possible_values is not None:
        if input_value not in possible_values:  # pragma: no cover
            raise ValueError(
                "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Provide a value from: " + str(possible_values) + ".")
    return True 

def load_tier_table(self, session: Session):
        """
        Load the tier table with values from the list.

        :return:
        """
        for tier_ind in TIER_LIST:
            tier_key = f'{tier_ind}'
            tier_descr = f'Tier {tier_key}'
            try:
                session.add(self.LocTier(loc_tier=tier_key,
                                         loc_tier_descr=tier_descr))
                session.commit()
            except IntegrityError:
                print(f'Tier {tier_ind} already exists')
                session.rollback()

        for record in session.query(self.LocTier):
            print(f'Tier: {record.loc_tier}  {record.loc_tier_descr} ') 

def load_bin_table(self, session: Session):
        """
        Load the bin table with values from min to max.

        :return:
        """
        for bin_ind in range(BIN_MIN, BIN_MAX + 1):
            bin_key = f'{bin_ind:02}'
            bin_descr = f'Bin {bin_key}'
            try:
                session.add(self.LocBin(loc_bin=bin_key,
                                        loc_bin_descr=bin_descr))
                session.commit()
            except IntegrityError:
                print(f'Bin {bin_ind} already exists')
                session.rollback()

        for record in session.query(self.LocBin):
            print(f'Bin: {record.loc_bin}  {record.loc_bin_descr} ') 

def load_row_table(self, session: Session):
        """
        Load the row table with values from min to max.

        :session:
        :return:
        """
        for row_ind in range(ROW_MIN, ROW_MAX + 1):
            row_key = f'{row_ind:02}'
            row_descr = f'Row {row_key}'
            try:
                session.add(self.LocRow(loc_row=row_key,
                                        loc_row_descr=row_descr))
                session.commit()
            except IntegrityError:
                print(f'Row {row_ind} already exists')
                session.rollback()

        for record in session.query(self.LocRow):
            print(f'Row: {record.loc_row}  {record.loc_row_descr} ')

        return 

def test_fsdir_missing_space(tmp_path):
    fshome = os.environ["FREESURFER_HOME"]

    # fsaverage2 doesn't exist in source or destination, so can't copy
    with pytest.raises(FileNotFoundError):
        bintfs.BIDSFreeSurferDir(
            derivatives=str(tmp_path), spaces=["fsaverage2"], freesurfer_home=fshome
        ).run()

    subjects_dir = tmp_path / "freesurfer"

    # If fsaverage2 exists in the destination directory, no error is thrown
    Path.mkdir(subjects_dir / "fsaverage2")
    bintfs.BIDSFreeSurferDir(
        derivatives=str(tmp_path), spaces=["fsaverage2"], freesurfer_home=fshome
    ).run() 

def test_fsdir(tmp_path, spaces):
    fshome = os.environ["FREESURFER_HOME"]
    subjects_dir = tmp_path / "freesurfer"

    # Verify we're starting clean
    for space in spaces:
        if space.startswith("fsaverage"):
            assert not Path.exists(subjects_dir / space)

    # Run three times to check idempotence
    # Third time force an overwrite
    for overwrite_fsaverage in (False, False, True):
        res = bintfs.BIDSFreeSurferDir(
            derivatives=str(tmp_path),
            spaces=spaces,
            freesurfer_home=fshome,
            overwrite_fsaverage=overwrite_fsaverage,
        ).run()
        assert res.outputs.subjects_dir == str(subjects_dir)

        for space in spaces:
            if space.startswith("fsaverage"):
                assert Path.exists(subjects_dir / space) 

def initializeTestDefinition(self):
        steps = []
        try:
            with open(self.csvFileName+".csv") as csvfile:
                reader = csv.DictReader(csvfile)
                for row in reader: 
                    index =0
                    for col in row:
                        expression = Path(os.path.join(self.confFile.getElementsByTagName("jsonsRepoPath")[0].firstChild.data,row[col]))
                        #pathExpected = Path((str(Path(__file__).parents[3]) + str(expression)))
                        pathExpected = os.path.join(str(self.dirPath),str(expression).rstrip())
                        if row[col]!="":
                            if Path.exists(Path(pathExpected)) :
                                steps.append(Step(row[col],index))
                                index+=1
                            else:
                                raise Exception 
            return steps
        except Exception as e:
            raise IOError("the file " + self.csvFileName + " or the jsons file inside the csv rows not found") 

def endFile(self):
        if self.errFile:
            self.errFile.flush()
        if self.warnFile:
            self.warnFile.flush()
        if self.diagFile:
            self.diagFile.flush()
        self.outFile.flush()
        if self.outFile != sys.stdout and self.outFile != sys.stderr:
            self.outFile.close()

        # On successfully generating output, move the temporary file to the
        # target file.
        if self.genOpts.filename is not None:
            if sys.platform == 'win32':
                directory = Path(self.genOpts.directory)
                if not Path.exists(directory):
                    os.makedirs(directory)
            shutil.move(self.outFile.name, self.genOpts.directory + '/' + self.genOpts.filename)
        self.genOpts = None 

def test_logging_failing_dimension_sql(tmpdir, sql_error):
    expected_directory = Path(tmpdir) / "queries"
    expected_directory.mkdir(exist_ok=True)

    log_sql_error(
        sql_error.model,
        sql_error.explore,
        sql_error.metadata["sql"],
        tmpdir,
        sql_error.metadata["dimension"],
    )
    expected_path = expected_directory / "eye_exam__users__users_age.sql"

    assert Path.exists(expected_path)
    with expected_path.open("r") as file:
        content = file.read()
    assert content == "SELECT age FROM users WHERE 1=2 LIMIT 1" 

def get_mathlib_olean(self) -> None:
        """Get precompiled mathlib oleans for this project."""
        # Just in case the user broke the workflow (for instance git clone
        # mathlib by hand and then run `leanproject get-cache`)
        if not (self.directory/'leanpkg.path').exists():
            self.run(['leanpkg', 'configure'])
        try:
            archive = get_mathlib_archive(self.mathlib_rev, self.cache_url,
                                           self.force_download, self.repo)
        except (EOFError, shutil.ReadError):
            log.info('Something wrong happened with the olean archive. '
                     'I will now retry downloading.')
            archive = get_mathlib_archive(self.mathlib_rev, self.cache_url,
                                          True, self.repo)
        self.clean_mathlib()
        self.mathlib_folder.mkdir(parents=True, exist_ok=True)
        unpack_archive(archive, self.mathlib_folder)
        # Let's now touch oleans, just in case
        touch_oleans(self.mathlib_folder) 

def clean_mathlib(self, force: bool = False) -> None:
        """Restore git sanity in mathlib"""
        if self.is_mathlib and (not self.is_dirty or force):
            self.repo.head.reset(working_tree=True)
            return
        if self.mathlib_folder.exists():
            mathlib = Repo(self.mathlib_folder)
            mathlib.head.reset(working_tree=True)
            mathlib.git.clean('-fd')
        else:
            self.run_echo(['leanpkg', 'configure']) 

def validate_file_list_values(file_list, no_of_epochs):
    if file_list is None:  # pragma: no cover
        raise ValueError("No value supplied for input file list: " + str(file_list))

    files = parse_namelist(file_list)

    for f in files:
        if not Path(f).exists():  # pragma: no cover
            raise ConfigException(f"{f} does not exist")
        else:
            matches = extract_epochs_from_filename(filename_with_epochs=f)
            if len(matches) < no_of_epochs:  # pragma: no cover
                raise ConfigException(f"the number of epochs in {f} names are less the required number: {no_of_epochs}") 

def create_dir(name):
    if not Path.exists(name):
        Path.mkdir(name) 

def mock_affinity_map_exists(mocker):
    mocker.patch('pathlib.Path.exists', return_value=True)
    mocker.patch('os.path.exists', return_value=True)
    mocker.patch('pathlib.Path.is_file', return_value=True)

    def side_effect(filename):
        if filename == './affinity_map.yml':
            return True
        else:
            Path.exists(filename)
    mocker.side_effect = side_effect 

def get_next_box_number(self) -> (str, int):
        """
        Search for the next box number to go on a label.

        :return:
        """
        next_box_number = self.box_start
        available_count = 0
        while available_count < self.label_count:
            box_label = f'BOX{next_box_number:05}'
            debug(f'Attempting to get {box_label}')
            sel_box_stm = select([self.box]).where(
                self.box.c.box_number == box_label)
            # box_found = exists(sel_box_stm)
            # exist_stm = exists().where(self.box.c.box_number == box_label)
            result = self.con.execute(sel_box_stm)
            debug(f'Search result: {result.rowcount}')
            box = result.fetchone()
            if not box:
                # found a hole in the numbers
                available_count += 1
                debug(f'{box_label} not found - using for label')
                yield (box_label, next_box_number)
            else:
                result.close()
            next_box_number += 1
        return 

def get_mathlib_archive(rev: str, url:str = '', force: bool = False,
                        repo: Optional[Repo] = None) -> Path:
    """Download a mathlib archive for revision rev into .mathlib

    Return the archive Path. Will raise LeanDownloadError if nothing works.
    """

    # we check for xz archives first
    fnames = [rev + '.tar.xz', rev + '.tar.gz', rev + '.tar.bz2']
    paths = [DOT_MATHLIB/fname for fname in fnames]
    if not force:
        log.info('Looking for local mathlib oleans')
        for path in paths:
            if path.exists():
                log.info('Found local mathlib oleans')
                return path
    log.info('Looking for remote mathlib oleans')
    for fname, path in zip(fnames, paths):
        try:
            base_url = url or get_download_url()
            download(base_url+fname, path)
            log.info('Found mathlib oleans at '+base_url)
            return path
        except LeanDownloadError:
            pass
    log.info('Looking for GitHub mathlib oleans')
    # nightlies will only store gz archives
    path = DOT_MATHLIB / (rev + '.tar.gz')
    download(nightly_url(rev, repo), path)
    log.info('Found GitHub mathlib oleans')
    return path 

def delete_zombies(dir: Path) -> None:
    for path in dir.glob('**/*.olean'):
        if not path.with_suffix('.lean').exists():
            log.info('deleting zombie {} ...'.format(str(path)))
            path.unlink() 

def test_fetch_credentials_no_dir(tmpdir):
    executor.Locker.LOCKDIR = str(tmpdir)
    agent.CREDENTIALS_PATH = str(tmpdir / 'notexist')
    file_path1 = tmpdir / 'notexist' / 'name1.json'
    file_path2 = tmpdir / 'notexist' / 'name2.json'

    mock_resp = mock.Mock()
    mock_resp.raise_status = 200
    mock_resp.json = mock.Mock(
        return_value=[
            {'name': 'name1', 'data': {'key1': 'v1'}},
            {'name': 'name2', 'data': {'key1': 'v21'}}
        ]
    )
    mock_resp.return_value.ok = True
    with mock.patch('agent.logger'), \
            mock.patch('agent.can_read_cert') as cr, \
            mock.patch('requests.request') as req:

        cr.return_value = True
        req.return_value = mock_resp
        mock_resp.return_value.ok = True
        agent.fetch_credentials(False)

        assert Path.exists(file_path1)
        assert Path.exists(file_path2)
        with open(str(file_path1)) as f:
            assert json.load(f) == {"key1": "v1"}

        with open(str(file_path2)) as f:
            assert json.load(f) == {"key1": "v21"} 

def test_inject_skullstrip(tmp_path):
    t1_mgz = tmp_path / "sub-01" / "mri" / "T1.mgz"
    t1_mgz.parent.mkdir(parents=True)
    # T1.mgz images are uint8
    nb.MGHImage(np.ones((5, 5, 5), dtype=np.uint8), np.eye(4)).to_filename(str(t1_mgz))

    mask_nii = tmp_path / "mask.nii.gz"
    # Masks may be in a different space (and need resampling), but should be boolean,
    # or uint8 in NIfTI
    nb.Nifti1Image(np.ones((6, 6, 6), dtype=np.uint8), np.eye(4)).to_filename(
        str(mask_nii)
    )

    FSInjectBrainExtracted(
        subjects_dir=str(tmp_path), subject_id="sub-01", in_brain=str(mask_nii)
    ).run()

    assert Path.exists(tmp_path / "sub-01" / "mri" / "brainmask.auto.mgz")
    assert Path.exists(tmp_path / "sub-01" / "mri" / "brainmask.mgz")

    # Run a second time to hit "already exists" condition
    FSInjectBrainExtracted(
        subjects_dir=str(tmp_path), subject_id="sub-01", in_brain=str(mask_nii)
    ).run() 

def mk_cache(self, force: bool = False) -> None:
        """Cache oleans for this project."""
        if self.is_dirty and not force:
            raise LeanDirtyRepo
        if not self.rev:
            raise ValueError('This project has no git commit.')
        tgt_folder = DOT_MATHLIB if self.is_mathlib else self.directory/'_cache'
        tgt_folder.mkdir(exist_ok=True)
        archive = tgt_folder/(str(self.rev) + '.tar.bz2')
        if archive.exists() and not force:
            log.info('Cache for revision {} already exists'.format(self.rev))
            return
        pack(self.directory, filter(Path.exists, [self.src_directory, self.directory/'test']),
             archive) 

def get_cache(self, force: bool = False, url:str = '') -> None:
        """Tries to get olean cache.

        Will raise LeanDownloadError or FileNotFoundError if no archive exists.
        """
        if self.is_dirty and not force:
            raise LeanDirtyRepo
        if self.is_mathlib:
            self.get_mathlib_olean()
        else:
            unpack_archive(self.directory/'_cache'/(str(self.rev)+'.tar.bz2'),
                           self.directory) 

def clean(self) -> None:
        src_dir = self.directory/self.pkg_config['path']
        test_dir = self.directory/'test'
        if src_dir.exists():
            clean(src_dir)
        else:
            raise InvalidLeanProject(
                "Directory {} specified by 'path' does not exist".format(src_dir))
        if test_dir.exists():
            clean(test_dir) 

def makeDir(self, path):
        """Create a directory, if not already done.

        Generally called from derived generators creating hierarchies."""
        self.logMsg('diag', 'OutputGenerator::makeDir(' + path + ')')
        if path not in self.madeDirs:
            # This can get race conditions with multiple writers, see
            # https://stackoverflow.com/questions/273192/
            if not os.path.exists(path):
                os.makedirs(path)
            self.madeDirs[path] = None 

def delete_zombies(self) -> None:
        src_dir = self.directory/self.pkg_config['path']
        test_dir = self.directory/'test'
        if src_dir.exists():
            delete_zombies(src_dir)
        else:
            raise InvalidLeanProject(
                "Directory {} specified by 'path' does not exist".format(src_dir))
        if test_dir.exists():
            delete_zombies(test_dir) 

def test_fetch_device_metadata(tmpdir):
    executor.Locker.LOCKDIR = str(tmpdir)
    json3_path_str = str(tmpdir / 'name3.json')
    json3_path = Path(json3_path_str)
    json3_path.write_text('nonzero')
    agent.SECRET_DEV_METADATA_PATH = str(json3_path_str)

    mock_resp = mock.Mock()
    mock_resp.raise_status = 200
    mock_resp.json = mock.Mock(
        return_value={
            'manufacturer': 'Raspberry Pi',
            'device_id': '7fe5ef257a7a4ee38841a5f8bf672791.d.wott-dev.local',
            'string': 'test string value',
            'array': [1, 2, 3, 4, 5, 'penelopa'],
            'test': 'value',
            'model': 'Pi 3 Model B+'
        }
    )
    mock_resp.return_value.ok = True
    with mock.patch('agent.can_read_cert') as cr, \
            mock.patch('requests.request') as req, \
            mock.patch('agent.logger'), \
            mock.patch('os.chmod') as chm:

        cr.return_value = True
        req.return_value = mock_resp
        mock_resp.return_value.ok = True
        agent.fetch_device_metadata(False, agent.logger)

        assert Path.exists(json3_path)

        with open(json3_path_str) as f:
            assert json.load(f) == {
                'manufacturer': 'Raspberry Pi',
                'device_id': '7fe5ef257a7a4ee38841a5f8bf672791.d.wott-dev.local',
                'string': 'test string value',
                'array': [1, 2, 3, 4, 5, 'penelopa'],
                'test': 'value',
                'model': 'Pi 3 Model B+'
            }

        chm.assert_has_calls([
            mock.call(json3_path_str, 0o600),
        ]) 

def load_location_table(self, session: Session):
        """
        Construct location records from the row/bin/tier records.

        The location code consists of the row code, bin code, and tier code
        jammed together into a six character id.

        :return:
        """
        for my_row in session.query(self.LocRow):
            for my_bin in session.query(self.LocBin):
                for my_tier in session.query(self.LocTier):
                    loc_code = (
                        f'{my_row.loc_row}{my_bin.loc_bin}{my_tier.loc_tier}'
                    )
                    loc_descr = (
                        f'Row {my_row.loc_row} '
                        f'Bin {my_bin.loc_bin} '
                        f'Tier {my_tier.loc_tier}'
                    )
                    loc_in_warehouse = True
                    try:
                        session.add(
                            self.Location(
                                loc_code=loc_code,
                                loc_descr=loc_descr,
                                loc_row_id=my_row.id,
                                loc_bin_id=my_bin.id,
                                loc_tier_id=my_tier.id,
                                loc_in_warehouse=loc_in_warehouse,
                            ))
                        session.commit()
                    except IntegrityError:
                        print(f'Location {loc_code} already exists')
                        session.rollback()

        # for loc_rec, row_rec, bin_rec, tier_rec in session.query(
        for loc_rec in session.query(
                self.Location).join(
                self.LocRow).join(
                self.LocBin).join(
                self.LocTier):
            print(
                f'Location: {loc_rec.loc_code} {loc_rec.loc_descr}'
                f'({loc_rec.locrow.loc_row_descr}/'
                f'{loc_rec.locbin.loc_bin_descr}/'
                f'{loc_rec.loctier.loc_tier_descr})'
            ) 

def start_logging(work_dir: Path, debug_name: str):
        """
        Establish the logging for all the other scripts.

        :param work_dir:
        :param debug_name:
        :return: (nothing)
        """

        # Set flag that no logging has been established
        logging_started = False

        # find our working directory and possible logging input file
        _workdir = work_dir
        _logfilename = debug_name

        # obtain the full path to the log information
        _debugConfig = _workdir / _logfilename

        # verify that the file exists before trying to open it
        if Path.exists(_debugConfig):
            try:
                #  get the logging params from yaml file and instantiate a log
                with open(_logfilename, 'r') as _logdictfd:
                    _logdict = yaml.load(_logdictfd, Loader=yaml.SafeLoader)
                logging.config.dictConfig(_logdict)
                logging_started = True
            except Exception as xcp:
                print(f'The file {_debugConfig} exists, but does not contain '
                      f'appropriate logging directives.')
                raise ValueError('Invalid logging directives.')
        else:
            print(f'Logging directives file {_debugConfig} either not '
                  f'specified or not found')

        if not logging_started:
            # set up minimal logging
            _logfilename = 'debuginfo.txt'
            _debugConfig = _workdir / _logfilename
            logging.basicConfig(filename='debuginfo.txt', level=logging.INFO,
                                filemode='w')
            print(f'Minimal logging established to {_debugConfig}')

        # start logging
        global log
        log = logging.getLogger(__name__)
        logging.info(f'Logging started: working directory is {_workdir}')
        return 

def start_logging(self, work_dir: Path, debug_name: str):
        """
        Establish the logging for all the other scripts.

        :param work_dir:
        :param debug_name:
        :return: (nothing)
        """

        # Set flag that no logging has been established
        logging_started = False

        # find our working directory and possible logging input file
        _workdir = work_dir
        _logfilename = debug_name

        # obtain the full path to the log information
        _debugConfig = _workdir / _logfilename

        # verify that the file exists before trying to open it
        if Path.exists(_debugConfig):
            try:
                #  get the logging params from yaml file and instantiate a log
                with open(_logfilename, 'r') as _logdictfd:
                    _logdict = yaml.load(_logdictfd, Loader=yaml.SafeLoader)
                logging.config.dictConfig(_logdict)
                logging_started = True
            except Exception as xcp:
                print(f'The file {_debugConfig} exists, but does not contain '
                      f'appropriate logging directives.')
                raise ValueError('Invalid logging directives.')
        else:
            print(f'Logging directives file {_debugConfig} either not '
                  f'specified or not found')

        if not logging_started:
            # set up minimal logging
            _logfilename = 'debuginfo.txt'
            _debugConfig = _workdir / _logfilename
            logging.basicConfig(filename='debuginfo.txt', level=logging.INFO,
                                filemode='w')
            print(f'Minimal logging established to {_debugConfig}')

        # start logging
        global log
        log = logging.getLogger(__name__)
        logging.info(f'Logging started: working directory is {_workdir}')

        # sset confirmed working directory to pass on to target class
        self.working_dir = _workdir
        return 

def list_decls(self) -> Dict[str, DeclInfo]:
        """Collect declarations seen from this project, as a dictionary of
        DeclInfo"""
        all_exists = (self.src_directory/'all.lean').exists()
        list_decls_lean = self.src_directory/'list_decls.lean'
        try:
            list_decls_lean.unlink()
        except FileNotFoundError:
            pass

        print('Gathering imports')
        self.make_all()
        imports = (self.src_directory/'all.lean').read_text()
        decls_lean = (Path(__file__).parent/'decls.lean').read_text()
        list_decls_lean.write_text(imports+decls_lean)
        print('Collecting declarations')
        self.run(['lean', '--run', str(list_decls_lean)])
        data = yaml.safe_load((self.directory/'decls.yaml').open())
        list_decls_lean.unlink()
        if not all_exists:
            (self.src_directory/'all.lean').unlink()
        decls = dict()
        for name, val in data.items():
            fname = val['File']
            line = val['Line']
            if fname is None or line is None:
                continue
            path = Path(fname)
            if '_target' in fname:
                path = path.relative_to(self.directory/'_target'/'deps')
                origin = path.parts[0]
                path = path.relative_to(Path(origin)/'src')
            elif '.elan' in fname:
                origin = 'core'
                parts = path.parts
                path = Path('/'.join(parts[parts.index('.elan')+7:]))
            else:
                origin = self.name
                path = path.relative_to(self.src_directory)
            decls[name] = DeclInfo(origin, path, int(val['Line']))

        return decls