Python portalocker.Lock() Examples

def step(self):
    actions = self.get_action(np.stack(self.obs))
    self.first = False
    [pipe.send(action) for pipe, action in zip(self.agent_pipes, actions)]
    next_obs, rewards, dones, resets = list(zip(*[pipe.recv() for pipe in self.agent_pipes]))

    frames = list(zip(self.obs, next_obs, actions, rewards, dones))

    self.obs = [o if resets[i] is False else self.agent_pipes[i].recv() for i, o in enumerate(next_obs)]

    for i, (t,r,reset) in enumerate(zip(self.total_rewards, rewards, resets)):
      if reset:
        self.total_rewards[i] = 0.
        if self.evaluation and self.loaded_policy:
          with portalocker.Lock(self.log_path+'.greedy.csv', mode="a") as f: f.write("%2f,%d,%d,%2f\n" % (self.hours, self.epoch, self.frame_total, t+r))

      else:
        self.total_rewards[i] = t + r

    if self.evaluation and np.any(resets): self.reload()

    self.rollout_i += 1
    return frames 

def step(self):
    actions = self.get_action(np.stack(self.obs))
    self.first = False
    [pipe.send(action) for pipe, action in zip(self.agent_pipes, actions)]
    next_obs, rewards, dones, resets = list(zip(*[pipe.recv() for pipe in self.agent_pipes]))

    frames = list(zip(self.obs, next_obs, actions, rewards, dones))

    self.obs = [o if resets[i] is False else self.agent_pipes[i].recv() for i, o in enumerate(next_obs)]

    for i, (t,r,reset) in enumerate(zip(self.total_rewards, rewards, resets)):
      if reset:
        self.total_rewards[i] = 0.
        if self.evaluation and self.loaded_policy:
          with portalocker.Lock(self.log_path+'.greedy.csv', mode="a") as f: f.write("%2f,%d,%d,%2f\n" % (self.hours, self.epoch, self.frame_total, t+r))

      else:
        self.total_rewards[i] = t + r

    if self.evaluation and np.any(resets): self.reload()

    self.rollout_i += 1
    return frames 

def step(self):
    actions = self.get_action(np.stack(self.obs))
    self.first = False
    [pipe.send(action) for pipe, action in zip(self.agent_pipes, actions)]
    next_obs, rewards, dones, resets = list(zip(*[pipe.recv() for pipe in self.agent_pipes]))

    frames = list(zip(self.obs, next_obs, actions, rewards, dones))

    self.obs = [o if resets[i] is False else self.agent_pipes[i].recv() for i, o in enumerate(next_obs)]

    for i, (t,r,reset) in enumerate(zip(self.total_rewards, rewards, resets)):
      if reset:
        self.total_rewards[i] = 0.
        if self.evaluation and self.loaded_policy:
          with portalocker.Lock(self.log_path+'.greedy.csv', mode="a") as f: f.write("%2f,%d,%d,%2f\n" % (self.hours, self.epoch, self.frame_total, t+r))

      else:
        self.total_rewards[i] = t + r

    if self.evaluation and np.any(resets): self.reload()

    self.rollout_i += 1
    return frames 

def write_file(filename, obj):
    file_data = 0
    while file_data is not None:
        lock = portalocker.Lock(filename, mode='a+b', flags=portalocker.LOCK_EX)
        lock.acquire()
        fh = lock.fh
        fh.seek(0)
        if len(fh.read()) is 0:
            file_data = None
        else:
            fh.seek(0)
            file_data = pickle.load(fh)
        if file_data is None:
            clear_file(fh)
            pickle.dump(obj, fh)
        lock.release() 

def getPath(uuid):
    try:
        config.logger.debug("register:getPath(" + str(uuid) + ")")
        register_semaphore.acquire(1)
        with portalocker.Lock(uuid_cache_path, timeout=1) as _:
            with shelve.open(uuid_cache_path) as db:
                try:
                    return str(db[uuid])
                except:
                    return None
    except Exception as e:
        config.logger.error("roast: Exception in getPath() %s",e)
        return None
    finally:
        if register_semaphore.available() < 1:
            register_semaphore.release(1) 

# scanns all .alog files for uuids and registers them in the cache 

def setAccount(account_id):
    try:
        config.logger.debug("account:setAccount(" + str(account_id) + ")")
        account_cache_semaphore.acquire(1)
        with portalocker.Lock(account_cache_lock_path, timeout=1) as _:
            with shelve.open(account_cache_path) as db:
                if account_id in db:
                    return db[account_id]
                else:
                    new_nr = len(db)
                    db[account_id] = new_nr
                    return new_nr
    except Exception as e:
        import sys
        _, _, exc_tb = sys.exc_info()
        config.logger.error("account: Exception in setAccount() line %s: %s",exc_tb.tb_lineno,e)
        return None
    finally:
        if account_cache_semaphore.available() < 1:
            account_cache_semaphore.release(1) 

def handle_submit(self, sender):
        import portalocker
        with portalocker.Lock(self.results_filename, "a+") as g:
            g.write("%s::%s::%s::%s\n" % (self.id, getpass.getuser(), datetime.datetime.today(), sender.description))
            g.flush()
            os.fsync(g.fileno())
        self.output.clear_output()
        with self.output:
            print("Received: " + sender.description) 

def save(cls, apps_dir=None):
        with portalocker.Lock(
                cls.get_apps_json_path(apps_dir=apps_dir), 'w',
                portalocker.LOCK_EX) as jf:
            data = CartoviewApp.objects.to_json()
            jf.write(data) 

def load(cls, apps_dir=None):
        if os.path.exists(cls.get_apps_json_path()):
            with portalocker.Lock(
                    cls.get_apps_json_path(apps_dir=apps_dir), 'r',
                    portalocker.LOCK_EX) as jf:
                data = jf.read()
                CartoviewApp.objects.from_json(data) 

def addPath(uuid,path):
    try:
        config.logger.debug("register:setPath(" + str(uuid) + "," + str(path) + ")")
        register_semaphore.acquire(1)
        with portalocker.Lock(uuid_cache_path, timeout=1) as _:
            with shelve.open(uuid_cache_path) as db:
                db[uuid] = str(path)
    except Exception as e:
        config.logger.error("roast: Exception in addPath() %s",e)
    finally:
        if register_semaphore.available() < 1:
            register_semaphore.release(1)  
    
# returns None if given uuid is not registered, otherwise the registered path 

def addSync(uuid,modified_at):
    try:
        config.logger.debug("sync:addSync(" + str(uuid) + "," + str(modified_at) + ")")
        sync_cache_semaphore.acquire(1)
        with portalocker.Lock(getSyncPath(lock=True), timeout=1) as _:
            with shelve.open(getSyncPath()) as db:
                db[uuid] = modified_at
    except Exception as e:
        config.logger.error("sync: Exception in addSync() %s",e)
    finally:
        if sync_cache_semaphore.available() < 1:
            sync_cache_semaphore.release(1)  
    
# returns None if given uuid is not registered for syncing, otherwise the last modified_at timestamp in EPOC milliseconds 

def read_file(filename):
    lock = portalocker.Lock(filename, mode='a+b', flags=portalocker.LOCK_EX)
    lock.acquire()
    fh = lock.fh
    fh.seek(0)
    if len(fh.read()) is 0:
        file_data = None
    else:
        fh.seek(0)
        file_data = pickle.load(fh)
    lock.release()
    return file_data 

def consume_file(filename):
    lock = portalocker.Lock(filename, mode='a+b', flags=portalocker.LOCK_EX)
    lock.acquire()
    fh = lock.fh
    fh.seek(0)
    if len(fh.read()) is 0:
        file_data = None
    else:
        fh.seek(0)
        file_data = pickle.load(fh)
    if file_data is not None:
        clear_file(fh)
        pickle.dump(None, fh)
    lock.release()
    return file_data 

def __init__(self,
                 num_procs,
                 dirname='file_comm',
                 worker_queue_file='worker_queue',
                 worker_results_prefix='worker_results_'):
        # make directory where communication files are created
        try:
            os.makedirs(dirname)
        except OSError:
            pass

        # claim a rank for the process
        lock = portalocker.Lock(os.path.join(dirname, 'init'),
                                mode='a+',
                                flags=portalocker.LOCK_EX)
        lock.acquire()
        fh = lock.fh
        fh.seek(0)
        curnum = fh.read()
        if len(curnum) is 0:
            rank = 0
        else:
            rank = int(curnum)

        if rank >= num_procs:
            raise ValueError('Number of processes > the number of workers')
        fh.seek(0)
        fh.truncate(0)
        fh.write(str(rank + 1))
        lock.release()

        super(FileCommunicator, self).__init__(num_procs - 1, rank)
        self.worker_queue_file = os.path.join(dirname, worker_queue_file)
        self.worker_results_prefix = os.path.join(dirname,
                                                  worker_results_prefix)
        self.done = False 

def write(self, file_or_path, append=False, timeout=10):
        """
        Write Smother results to a file.

        Parameters
        ----------
        fiile_or_path : str
            Path to write report to
        append : bool
            If True, read an existing smother report from `outpath`
            and combine it with this file before writing.
        timeout : int
            Time in seconds to wait to acquire a file lock, before
            raising an error.

        Note
        ----
        Append mode is atomic when file_or_path is a path,
        and can be safely run in a multithreaded or
        multiprocess test environment.

        When using `parallel_mode`, file_or_path is given a unique
        suffix based on the machine name and process id.
        """
        if isinstance(file_or_path, six.string_types):
            if self.coverage:
                file_or_path = get_smother_filename(
                    file_or_path, self.coverage.config.parallel)

            outfile = Lock(
                file_or_path, mode='a+',
                timeout=timeout,
                fail_when_locked=False
            )
        else:
            outfile = noclose(file_or_path)

        with outfile as fh:

            if append:
                fh.seek(0)
                try:
                    other = Smother.load(fh)
                except ValueError:  # no smother data
                    pass
                else:
                    self |= other

            fh.seek(0)
            fh.truncate()  # required to overwrite data in a+ mode
            json.dump(self.data, fh) 

def hook(
    wakeups: List[Wakeup],
    wakeup_delta: float,
    wakeup_fn: Callable[[datetime.datetime], None],
    woke_up_file: str,
    lock_file: str,
    lock_timeout: float,
) -> None:
    """Installs wake ups and notifies the daemon before suspending.

    Args:
        wakeups:
            set of wakeup checks to use for determining the wake up time
        wakeup_delta:
            The amount of time in seconds to wake up before an event
        wakeup_fn:
            function to call with the next wake up time
        woke_up_file:
            location of the file that instructs the daemon that the system just
            woke up
        lock_file:
            path of a file used for locking modifications to the `woke_up_file`
            to ensure consistency
        lock_timeout:
            time in seconds to wait for acquiring the lock file
    """
    _logger.info("Pre-suspend hook starting, trying to acquire lock")
    try:
        with portalocker.Lock(lock_file, timeout=lock_timeout):
            _logger.debug("Hook acquired lock")

            _logger.debug("Hook executing with configured wake ups: %s", wakeups)
            wakeup_at = execute_wakeups(
                wakeups, datetime.datetime.now(datetime.timezone.utc), _logger
            )
            _logger.debug("Hook next wake up at %s", wakeup_at)

            if wakeup_at:
                wakeup_at -= datetime.timedelta(seconds=wakeup_delta)
                _logger.info("Scheduling next wake up at %s", wakeup_at)
                wakeup_fn(wakeup_at)
            else:
                _logger.info("No wake up required. Terminating")

            # create the just woke up file
            pathlib.Path(woke_up_file).touch()
    except portalocker.LockException:
        _logger.warning(
            "Hook unable to acquire lock. Not informing daemon.", exc_info=True
        ) 

def loop(
    processor: Processor,
    interval: float,
    run_for: Optional[int],
    woke_up_file: str,
    lock_file: str,
    lock_timeout: float,
) -> None:
    """Run the main loop of the daemon.

    Args:
        processor:
            the processor to use for handling the suspension computations
        interval:
            the length of one iteration of the main loop in seconds
        run_for:
            if specified, run the main loop for the specified amount of seconds
            before terminating (approximately)
        woke_up_file:
            path of a file that marks that the system was sleeping since the
            last processing iterations
        lock_file:
            path of a file used for locking modifications to the `woke_up_file`
            to ensure consistency
        lock_timeout:
            time in seconds to wait for acquiring the lock file
    """

    start_time = datetime.datetime.now(datetime.timezone.utc)
    while (run_for is None) or (
        datetime.datetime.now(datetime.timezone.utc)
        < (start_time + datetime.timedelta(seconds=run_for))
    ):

        try:
            _logger.debug("New iteration, trying to acquire lock")
            with portalocker.Lock(lock_file, timeout=lock_timeout):
                _logger.debug("Acquired lock")

                just_woke_up = os.path.isfile(woke_up_file)
                if just_woke_up:
                    _logger.debug("Removing woke up file at %s", woke_up_file)
                    try:
                        os.remove(woke_up_file)
                    except FileNotFoundError:
                        _logger.warning("Just woke up file disappeared", exc_info=True)

                processor.iteration(
                    datetime.datetime.now(datetime.timezone.utc), just_woke_up
                )

        except portalocker.LockException:
            _logger.warning("Failed to acquire lock, skipping iteration", exc_info=True)

        time.sleep(interval) 

def get_model_file(name, root=os.path.join('~', '.encoding', 'models')):
    r"""Return location for the pretrained on local file system.

    This function will download from online model zoo when model cannot be found or has mismatch.
    The root directory will be created if it doesn't exist.

    Parameters
    ----------
    name : str
        Name of the model.
    root : str, default '~/.encoding/models'
        Location for keeping the model parameters.

    Returns
    -------
    file_path
        Path to the requested pretrained model file.
    """
    if name not in _model_sha1:
        from torchvision.models.resnet import model_urls
        if name not in model_urls:
            raise ValueError('Pretrained model for {name} is not available.'.format(name=name))
        root = os.path.expanduser(root)
        return download(model_urls[name],
                        path=root,
                        overwrite=True)
    file_name = '{name}-{short_hash}'.format(name=name, short_hash=short_hash(name))
    root = os.path.expanduser(root)
    if not os.path.exists(root):
        os.makedirs(root)

    file_path = os.path.join(root, file_name+'.pth')
    sha1_hash = _model_sha1[name]

    lockfile = os.path.join(root, file_name + '.lock')
    with portalocker.Lock(lockfile, timeout=300):
        if os.path.exists(file_path):
            if check_sha1(file_path, sha1_hash):
                return file_path
            else:
                print('Mismatch in the content of model file {} detected.' +
                      ' Downloading again.'.format(file_path))
        else:
            print('Model file {} is not found. Downloading.'.format(file_path))

        zip_file_path = os.path.join(root, file_name+'.zip')
        repo_url = os.environ.get('ENCODING_REPO', encoding_repo_url)
        if repo_url[-1] != '/':
            repo_url = repo_url + '/'
        download(_url_format.format(repo_url=repo_url, file_name=file_name),
                 path=zip_file_path,
                 overwrite=True)
        with zipfile.ZipFile(zip_file_path) as zf:
            zf.extractall(root)
        os.remove(zip_file_path)

        if check_sha1(file_path, sha1_hash):
            return file_path
        else:
            raise ValueError('Downloaded file has different hash. Please try again.') 

def import_try_install(package, extern_url=None):
    """Try import the specified package.
    If the package not installed, try use pip to install and import if success.

    Parameters
    ----------
    package : str
        The name of the package trying to import.
    extern_url : str or None, optional
        The external url if package is not hosted on PyPI.
        For example, you can install a package using:
         "pip install git+http://github.com/user/repo/tarball/master/egginfo=xxx".
        In this case, you can pass the url to the extern_url.

    Returns
    -------
    <class 'Module'>
        The imported python module.

    """
    import tempfile
    import portalocker
    lockfile = os.path.join(tempfile.gettempdir(), package + '_install.lck')
    with portalocker.Lock(lockfile):
        try:
            return __import__(package)
        except ImportError:
            try:
                from pip import main as pipmain
            except ImportError:
                from pip._internal import main as pipmain
                from types import ModuleType
                # fix for pip 19.3
                if isinstance(pipmain, ModuleType):
                    from pip._internal.main import main as pipmain

            # trying to install package
            url = package if extern_url is None else extern_url
            pipmain(['install', '--user', url])  # will raise SystemExit Error if fails

            # trying to load again
            try:
                return __import__(package)
            except ImportError:
                import sys
                import site
                user_site = site.getusersitepackages()
                if user_site not in sys.path:
                    sys.path.append(user_site)
                return __import__(package)
    return __import__(package)