Python portalocker.LOCK_EX Examples

def acquire(self):
        """ Acquire thread and file locks.  Re-opening log for 'degraded' mode.
        """
        # handle thread lock
        Handler.acquire(self)
        # Issue a file lock.  (This is inefficient for multiple active threads
        # within a single process. But if you're worried about high-performance,
        # you probably aren't using this log handler.)
        if self.stream_lock:
            # If stream_lock=None, then assume close() was called or something
            # else weird and ignore all file-level locks.
            if self.stream_lock.closed:
                # Daemonization can close all open file descriptors, see
                # https://bugzilla.redhat.com/show_bug.cgi?id=952929
                # Try opening the lock file again.  Should we warn() here?!?
                try:
                    self._open_lockfile()
                except Exception:
                    self.handleError(NullLogRecord())
                    # Don't try to open the stream lock again
                    self.stream_lock = None
                    return
            lock(self.stream_lock, LOCK_EX)
            # Stream will be opened as part by FileHandler.emit() 

def write_file(filename, obj):
    file_data = 0
    while file_data is not None:
        lock = portalocker.Lock(filename, mode='a+b', flags=portalocker.LOCK_EX)
        lock.acquire()
        fh = lock.fh
        fh.seek(0)
        if len(fh.read()) is 0:
            file_data = None
        else:
            fh.seek(0)
            file_data = pickle.load(fh)
        if file_data is None:
            clear_file(fh)
            pickle.dump(obj, fh)
        lock.release() 

def init_list(self, data=None):
        """
        Initialize asid_list file.
        :param data: list or a string
        :return:
        """
        file_path = os.path.join(running_path, '{sid}_list'.format(sid=self.sid))
        if not os.path.exists(file_path):
            if isinstance(data, list):
                with open(file_path, 'w') as f:
                    portalocker.lock(f, portalocker.LOCK_EX)
                    f.write(json.dumps({
                        'sids': {},
                        'total_target_num': len(data),
                    }))
            else:
                with open(file_path, 'w') as f:
                    portalocker.lock(f, portalocker.LOCK_EX)
                    f.write(json.dumps({
                        'sids': {},
                        'total_target_num': 1,
                    })) 

def list(self, data=None):
        file_path = os.path.join(running_path, '{sid}_list'.format(sid=self.sid))
        if data is None:
            with open(file_path, 'r') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                result = f.readline()
                return json.loads(result)
        else:
            with open(file_path, 'r+') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                result = f.read()
                if result == '':
                    result = {'sids': {}}
                else:
                    result = json.loads(result)
                result['sids'][data[0]] = data[1]
                f.seek(0)
                f.truncate()
                f.write(json.dumps(result)) 

def _try_store_in_file(self, request, response):
        try:
            if (not response.session_id or self._forget 
                or self._unchanged(response)):
                # self.clear_session_cookies()
                self.save_session_id_cookie()
                return False
            if response.session_new or not response.session_file:
                # Tests if the session sub-folder exists, if not, create it
                session_folder = os.path.dirname(response.session_filename)
                if not os.path.exists(session_folder): os.mkdir(session_folder)
                response.session_file = open(response.session_filename, 'wb')
                portalocker.lock(response.session_file, portalocker.LOCK_EX)
                response.session_locked = True
            if response.session_file:
                session_pickled = response.session_pickled or cPickle.dumps(self)
                response.session_file.write(session_pickled)
                response.session_file.truncate()
        finally:
            self._close(response)

        self.save_session_id_cookie()
        return True 

def _do_lock(self):
        if self.is_locked:
            return   # already locked... recursive?
        self._open_lockfile()
        if self.stream_lock:
            for i in range(10):
                # noinspection PyBroadException
                try:
                    lock(self.stream_lock, LOCK_EX)
                    self.is_locked = True
                    break
                except Exception:
                    continue
            else:
                raise RuntimeError("Cannot acquire lock after 10 attempts")
        else:
            self._console_log("No self.stream_lock to lock", stack=True) 

def __enter__(self) -> Optional[GpuDeviceType]:
        for gpu_id in self.candidates:
            lockfile_path = os.path.join(self.lock_dir, "sockeye.gpu{}.lock".format(gpu_id))
            try:
                lock_file = open(lockfile_path, 'w')
            except IOError:
                if errno.EACCES:
                    logger.warning("GPU {} is currently locked by a different process "
                                   "(Permission denied).".format(gpu_id))
                    continue
            try:
                # exclusive non-blocking lock
                portalocker.lock(lock_file, portalocker.LOCK_EX | portalocker.LOCK_NB)
                # got the lock, let's write our PID into it:
                lock_file.write("%d\n" % os.getpid())
                lock_file.flush()

                self._acquired_lock = True
                self.gpu_id = gpu_id
                self.lock_file = lock_file
                self.lockfile_path = lockfile_path

                logger.info("Acquired GPU {}.".format(gpu_id))

                return gpu_id
            except portalocker.LockException as e:
                # portalocker packages the original exception,
                # we dig it out and raise if unrelated to us
                if e.args[0].errno != errno.EAGAIN:  # pylint: disable=no-member
                    logger.error("Failed acquiring GPU lock.", exc_info=True)
                    raise e.args[0]
                else:
                    logger.debug("GPU {} is currently locked.".format(gpu_id))
        return None 

def updateparallelprogress(added_value):
    Done = False
    while (not Done):
        try:
            Done = True
            with open(glob.tmpfile, mode='r+') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                data = f.read().split(" ")
                parallel_total = float(data[0])
                log_every_percentage = float(data[1])
                current_parallel_progress = float(data[2])

                last_log = math.floor(100 * current_parallel_progress
                                      / parallel_total / log_every_percentage) * log_every_percentage
                current_parallel_progress += added_value
                new_log = math.floor(100 * current_parallel_progress
                                     / parallel_total / log_every_percentage) * log_every_percentage

                #for i in np.arange(last_log+log_every_percentage, new_log + log_every_percentage, log_every_percentage):
                #    log('{0}%'.format(int(i)))
                if new_log > last_log:
                    log('{0}%'.format(int(new_log)))

                f.seek(0)
                f.write('{0} {1} {2}'.format(parallel_total, log_every_percentage, current_parallel_progress))
                f.truncate()
        except:
            Done = False 

def release(self):
        """
        this function writes into cron.master the time when cron job
        was completed
        """
        if not self.master.closed:
            portalocker.lock(self.master, portalocker.LOCK_EX)
            logger.debug('WEB2PY CRON: Releasing cron lock')
            self.master.seek(0)
            (start, stop) = cPickle.load(self.master)
            if start == self.now:  # if this is my lock
                self.master.seek(0)
                cPickle.dump((self.now, time.time()), self.master)
            portalocker.unlock(self.master)
            self.master.close() 

def _open_shelve_and_lock(self):
        """Open and return a shelf object, obtaining an exclusive lock
        on self.locker first. Replaces the close method of the
        returned shelf instance with one that releases the lock upon
        closing."""

        storage = None
        locker = None
        locked = False
        try:
            locker = locker = open(self.locker_name, 'a')
            portalocker.lock(locker, portalocker.LOCK_EX)
            locked = True
            try:
                storage = shelve.open(self.shelve_name)
            except:
                logger.error('corrupted cache file %s, will try rebuild it'
                             % self.shelve_name)
                storage = None
            if storage is None:
                if os.path.exists(self.shelve_name):
                    os.unlink(self.shelve_name)
                storage = shelve.open(self.shelve_name)
            if not CacheAbstract.cache_stats_name in storage.keys():
                storage[CacheAbstract.cache_stats_name] = {
                    'hit_total': 0, 'misses': 0}
            storage.sync()
        except Exception, e:
            if storage:
                storage.close()
                storage = None
            if locked:
                portalocker.unlock(locker)
                locker.close()
            locked = False
            raise RuntimeError(
                'unable to create/re-create cache file %s' % self.shelve_name) 

def __init__(self, filename, mode='rb'):
        self.filename = filename
        self.mode = mode
        self.file = None
        if 'r' in mode:
            self.file = open(filename, mode)
            lock(self.file, LOCK_SH)
        elif 'w' in mode or 'a' in mode:
            self.file = open(filename, mode.replace('w', 'a'))
            lock(self.file, LOCK_EX)
            if not 'a' in mode:
                self.file.seek(0)
                self.file.truncate()
        else:
            raise RuntimeError("invalid LockedFile(...,mode)") 

def save(cls, apps_dir=None):
        with portalocker.Lock(
                cls.get_apps_json_path(apps_dir=apps_dir), 'w',
                portalocker.LOCK_EX) as jf:
            data = CartoviewApp.objects.to_json()
            jf.write(data) 

def load(cls, apps_dir=None):
        if os.path.exists(cls.get_apps_json_path()):
            with portalocker.Lock(
                    cls.get_apps_json_path(apps_dir=apps_dir), 'r',
                    portalocker.LOCK_EX) as jf:
                data = jf.read()
                CartoviewApp.objects.from_json(data) 

def __init__(self, filename, mode="rb"):
        self.filename = filename
        self.mode = mode
        self.file = None
        if "r" in mode:
            self.file = open_file(filename, mode)
            lock(self.file, LOCK_SH)
        elif "w" in mode or "a" in mode:
            self.file = open_file(filename, mode.replace("w", "a"))
            lock(self.file, LOCK_EX)
            if "a" not in mode:
                self.file.seek(0)
                self.file.truncate(0)
        else:
            raise RuntimeError("invalid LockedFile(...,mode)") 

def __init__(self,
                 num_procs,
                 dirname='file_comm',
                 worker_queue_file='worker_queue',
                 worker_results_prefix='worker_results_'):
        # make directory where communication files are created
        try:
            os.makedirs(dirname)
        except OSError:
            pass

        # claim a rank for the process
        lock = portalocker.Lock(os.path.join(dirname, 'init'),
                                mode='a+',
                                flags=portalocker.LOCK_EX)
        lock.acquire()
        fh = lock.fh
        fh.seek(0)
        curnum = fh.read()
        if len(curnum) is 0:
            rank = 0
        else:
            rank = int(curnum)

        if rank >= num_procs:
            raise ValueError('Number of processes > the number of workers')
        fh.seek(0)
        fh.truncate(0)
        fh.write(str(rank + 1))
        lock.release()

        super(FileCommunicator, self).__init__(num_procs - 1, rank)
        self.worker_queue_file = os.path.join(dirname, worker_queue_file)
        self.worker_results_prefix = os.path.join(dirname,
                                                  worker_results_prefix)
        self.done = False 

def acquire(self):
        """ Acquire thread and file locks. Also re-opening log file when running
        in 'degraded' mode. """
        # handle thread lock
        Handler.acquire(self)
        lock(self.stream_lock, LOCK_EX)
        if self.stream.closed:
            self._openFile(self.mode) 

def data(self, data=None):

        file_path = os.path.abspath(running_path + '/{sid}_data'.format(sid=self.sid))

        if data is None:
            with open(file_path) as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                result = f.readline()
            return json.loads(result)
        else:
            data = json.dumps(data, sort_keys=True)
            with open(file_path, 'w+') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                f.writelines(data) 

def status(self, data=None):
        file_path = os.path.join(running_path, '{sid}_status'.format(sid=self.sid))
        if data is None:
            with open(file_path) as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                result = f.readline()
            return json.loads(result)
        else:
            data = json.dumps(data)
            with open(file_path, 'w') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                f.writelines(data) 

def get(self, count: int=1, readOnly: bool=False, recurlvl=0):
        with open(self.FILE, "r+") as file:
            portalocker.lock(file, portalocker.LOCK_EX)
            ports = []
            while len(ports) < count:
                file.seek(0)
                port = int(file.readline())
                if readOnly:
                    return port
                port += 1
                if port > self.maxPort:
                    port = self.minPort
                file.seek(0)
                file.write(str(port))
                try:
                    checkPortAvailable(("", port))
                    ports.append(port)
                    self.logger.debug("new port dispensed: {}".format(port))
                except Exception:
                    if recurlvl < self.maxportretries:
                        self.logger.debug("port {} unavailable, trying again...".
                                          format(port))
                        recurlvl += 1
                    else:
                        self.logger.debug("port {} unavailable, max retries {} "
                                          "reached".
                                          format(port, self.maxportretries))
                        raise
            return ports 

def read_file(filename):
    lock = portalocker.Lock(filename, mode='a+b', flags=portalocker.LOCK_EX)
    lock.acquire()
    fh = lock.fh
    fh.seek(0)
    if len(fh.read()) is 0:
        file_data = None
    else:
        fh.seek(0)
        file_data = pickle.load(fh)
    lock.release()
    return file_data 

def consume_file(filename):
    lock = portalocker.Lock(filename, mode='a+b', flags=portalocker.LOCK_EX)
    lock.acquire()
    fh = lock.fh
    fh.seek(0)
    if len(fh.read()) is 0:
        file_data = None
    else:
        fh.seek(0)
        file_data = pickle.load(fh)
    if file_data is not None:
        clear_file(fh)
        pickle.dump(None, fh)
    lock.release()
    return file_data 

def acquire(self, startup=False):
        """
        returns the time when the lock is acquired or
        None if cron already running

        lock is implemented by writing a pickle (start, stop) in cron.master
        start is time when cron job starts and stop is time when cron completed
        stop == 0 if job started but did not yet complete
        if a cron job started within less than 60 seconds, acquire returns None
        if a cron job started before 60 seconds and did not stop,
        a warning is issue "Stale cron.master detected"
        """
        if sys.platform == 'win32':
            locktime = 59.5
        else:
            locktime = 59.99
        if portalocker.LOCK_EX is None:
            logger.warning('WEB2PY CRON: Disabled because no file locking')
            return None
        self.master = open(self.path, 'rb+')
        try:
            ret = None
            portalocker.lock(self.master, portalocker.LOCK_EX)
            try:
                (start, stop) = cPickle.load(self.master)
            except:
                (start, stop) = (0, 1)
            if startup or self.now - start > locktime:
                ret = self.now
                if not stop:
                    # this happens if previous cron job longer than 1 minute
                    logger.warning('WEB2PY CRON: Stale cron.master detected')
                logger.debug('WEB2PY CRON: Acquiring lock')
                self.master.seek(0)
                cPickle.dump((self.now, 0), self.master)
                self.master.flush()
        finally:
            portalocker.unlock(self.master)
        if not ret:
            # do this so no need to release
            self.master.close()
        return ret