Python portalocker.lock() Examples

def init_list(self, data=None):
        """
        Initialize asid_list file.
        :param data: list or a string
        :return:
        """
        file_path = os.path.join(running_path, '{sid}_list'.format(sid=self.sid))
        if not os.path.exists(file_path):
            if isinstance(data, list):
                with open(file_path, 'w') as f:
                    portalocker.lock(f, portalocker.LOCK_EX)
                    f.write(json.dumps({
                        'sids': {},
                        'total_target_num': len(data),
                    }))
            else:
                with open(file_path, 'w') as f:
                    portalocker.lock(f, portalocker.LOCK_EX)
                    f.write(json.dumps({
                        'sids': {},
                        'total_target_num': 1,
                    })) 

def release(self):
        """ Release file and thread locks. If in 'degraded' mode, close the
        stream to reduce contention until the log files can be rotated. """
        try:
            if self._rotateFailed:
                self._close()
        except Exception:
            self.handleError(NullLogRecord())
        finally:
            try:
                if self.stream_lock and not self.stream_lock.closed:
                    unlock(self.stream_lock)
            except Exception:
                self.handleError(NullLogRecord())
            finally:
                # release thread lock
                Handler.release(self) 

def acquire(self):
        """ Acquire thread and file locks.  Re-opening log for 'degraded' mode.
        """
        # handle thread lock
        Handler.acquire(self)
        # Issue a file lock.  (This is inefficient for multiple active threads
        # within a single process. But if you're worried about high-performance,
        # you probably aren't using this log handler.)
        if self.stream_lock:
            # If stream_lock=None, then assume close() was called or something
            # else weird and ignore all file-level locks.
            if self.stream_lock.closed:
                # Daemonization can close all open file descriptors, see
                # https://bugzilla.redhat.com/show_bug.cgi?id=952929
                # Try opening the lock file again.  Should we warn() here?!?
                try:
                    self._open_lockfile()
                except Exception:
                    self.handleError(NullLogRecord())
                    # Don't try to open the stream lock again
                    self.stream_lock = None
                    return
            lock(self.stream_lock, LOCK_EX)
            # Stream will be opened as part by FileHandler.emit() 

def initialize(self):
        if self.initialized:
            return
        else:
            self.initialized = True
        folder = self.folder
        request = self.request

        # Lets test if the cache folder exists, if not
        # we are going to create it
        folder = folder or os.path.join(request.folder, 'cache')

        if not os.path.exists(folder):
            os.mkdir(folder)

        ### we need this because of a possible bug in shelve that may
        ### or may not lock
        self.locker_name = os.path.join(folder, 'cache.lock')
        self.shelve_name = os.path.join(folder, 'cache.shelve') 

def _try_store_in_file(self, request, response):
        try:
            if (not response.session_id or self._forget 
                or self._unchanged(response)):
                # self.clear_session_cookies()
                self.save_session_id_cookie()
                return False
            if response.session_new or not response.session_file:
                # Tests if the session sub-folder exists, if not, create it
                session_folder = os.path.dirname(response.session_filename)
                if not os.path.exists(session_folder): os.mkdir(session_folder)
                response.session_file = open(response.session_filename, 'wb')
                portalocker.lock(response.session_file, portalocker.LOCK_EX)
                response.session_locked = True
            if response.session_file:
                session_pickled = response.session_pickled or cPickle.dumps(self)
                response.session_file.write(session_pickled)
                response.session_file.truncate()
        finally:
            self._close(response)

        self.save_session_id_cookie()
        return True 

def _do_lock(self):
        if self.is_locked:
            return   # already locked... recursive?
        self._open_lockfile()
        if self.stream_lock:
            for i in range(10):
                # noinspection PyBroadException
                try:
                    lock(self.stream_lock, LOCK_EX)
                    self.is_locked = True
                    break
                except Exception:
                    continue
            else:
                raise RuntimeError("Cannot acquire lock after 10 attempts")
        else:
            self._console_log("No self.stream_lock to lock", stack=True) 

def getLockFilename(self):
        """
        Decide the lock filename. If the logfile is file.log, then we use `.__file.lock` and
        not `file.log.lock`. This only removes the extension if it's `*.log`.

        :return: the path to the lock file.
        """
        if self.baseFilename.endswith(".log"):
            lock_file = self.baseFilename[:-4]
        else:
            lock_file = self.baseFilename
        lock_file += ".lock"
        lock_path, lock_name = os.path.split(lock_file)
        # hide the file on Unix and generally from file completion
        lock_name = ".__" + lock_name
        return os.path.join(lock_path, lock_name) 

def list(self, data=None):
        file_path = os.path.join(running_path, '{sid}_list'.format(sid=self.sid))
        if data is None:
            with open(file_path, 'r') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                result = f.readline()
                return json.loads(result)
        else:
            with open(file_path, 'r+') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                result = f.read()
                if result == '':
                    result = {'sids': {}}
                else:
                    result = json.loads(result)
                result['sids'][data[0]] = data[1]
                f.seek(0)
                f.truncate()
                f.write(json.dumps(result)) 

def get(self, count: int=1, readOnly: bool=False, recurlvl=0):
        with open(self.FILE, "r+") as file:
            portalocker.lock(file, portalocker.LOCK_EX)
            ports = []
            while len(ports) < count:
                file.seek(0)
                port = int(file.readline())
                if readOnly:
                    return port
                port += 1
                if port > self.maxPort:
                    port = self.minPort
                file.seek(0)
                file.write(str(port))
                try:
                    checkPortAvailable(("", port))
                    ports.append(port)
                    self.logger.debug("new port dispensed: {}".format(port))
                except Exception:
                    if recurlvl < self.maxportretries:
                        self.logger.debug("port {} unavailable, trying again...".
                                          format(port))
                        recurlvl += 1
                    else:
                        self.logger.debug("port {} unavailable, max retries {} "
                                          "reached".
                                          format(port, self.maxportretries))
                        raise
            return ports 

def emit(self, record):
        """
        Emit a record.

        Override from parent class to handle file locking for the duration of rollover and write.
        This also does the formatting *before* locks are obtained, in case the format itself does
        logging calls from within. Rollover also occurs while the lock is held.
        """
        # noinspection PyBroadException
        try:
            msg = self.format(record)
            try:
                self._do_lock()

                try:
                    if self.shouldRollover(record):
                        self.doRollover()
                except Exception as e:
                    self._console_log("Unable to do rollover: %s" % (e,), stack=True)
                    # Continue on anyway

                self.do_write(msg)

            finally:
                self._do_unlock()
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception:
            self.handleError(record) 

def _open_lockfile(self):
        # Use 'file.lock' and not 'file.log.lock' (Only handles the normal "*.log" case.)
        if self.baseFilename.endswith(".log"):
            lock_file = self.baseFilename[:-4]
        else:
            lock_file = self.baseFilename
        lock_file += ".lock"
        self.stream_lock = open(lock_file, "w") 

def lock(file, flags):
        try:
            fcntl.flock(file.fileno(), flags)
        except IOError, exc_value:
            # The exception code varies on different systems so we'll catch
            # every IO error
            raise LockException(*exc_value) 

def lock(file, flags):
        hfile = win32file._get_osfhandle(file.fileno())
        try:
            win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
        except pywintypes.error, exc_value:
            # error: (33, 'LockFileEx', 'The process cannot access the file because another process has locked a portion of the file.')
            if exc_value[0] == 33:
                raise LockException(LockException.LOCK_FAILED, exc_value[2])
            else:
                # Q:  Are there exceptions/codes we should be dealing with here?
                raise 

def updateparallelprogress(added_value):
    Done = False
    while (not Done):
        try:
            Done = True
            with open(glob.tmpfile, mode='r+') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                data = f.read().split(" ")
                parallel_total = float(data[0])
                log_every_percentage = float(data[1])
                current_parallel_progress = float(data[2])

                last_log = math.floor(100 * current_parallel_progress
                                      / parallel_total / log_every_percentage) * log_every_percentage
                current_parallel_progress += added_value
                new_log = math.floor(100 * current_parallel_progress
                                     / parallel_total / log_every_percentage) * log_every_percentage

                #for i in np.arange(last_log+log_every_percentage, new_log + log_every_percentage, log_every_percentage):
                #    log('{0}%'.format(int(i)))
                if new_log > last_log:
                    log('{0}%'.format(int(new_log)))

                f.seek(0)
                f.write('{0} {1} {2}'.format(parallel_total, log_every_percentage, current_parallel_progress))
                f.truncate()
        except:
            Done = False 

def release(self):
        """
        this function writes into cron.master the time when cron job
        was completed
        """
        if not self.master.closed:
            portalocker.lock(self.master, portalocker.LOCK_EX)
            logger.debug('WEB2PY CRON: Releasing cron lock')
            self.master.seek(0)
            (start, stop) = cPickle.load(self.master)
            if start == self.now:  # if this is my lock
                self.master.seek(0)
                cPickle.dump((self.now, time.time()), self.master)
            portalocker.unlock(self.master)
            self.master.close() 

def status(self, data=None):
        file_path = os.path.join(running_path, '{sid}_status'.format(sid=self.sid))
        if data is None:
            with open(file_path) as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                result = f.readline()
            return json.loads(result)
        else:
            data = json.dumps(data)
            with open(file_path, 'w') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                f.writelines(data) 

def data(self, data=None):

        file_path = os.path.abspath(running_path + '/{sid}_data'.format(sid=self.sid))

        if data is None:
            with open(file_path) as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                result = f.readline()
            return json.loads(result)
        else:
            data = json.dumps(data, sort_keys=True)
            with open(file_path, 'w+') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                f.writelines(data) 

def __init__(self, filename, mode='rb'):
        self.filename = filename
        self.mode = mode
        self.file = None
        if 'r' in mode:
            self.file = open(filename, mode)
            lock(self.file, LOCK_SH)
        elif 'w' in mode or 'a' in mode:
            self.file = open(filename, mode.replace('w', 'a'))
            lock(self.file, LOCK_EX)
            if not 'a' in mode:
                self.file.seek(0)
                self.file.truncate()
        else:
            raise RuntimeError("invalid LockedFile(...,mode)") 

def lock(file, flags):
        pass 

def lock(file, flags):
        fcntl.flock(file.fileno(), flags) 

def lock(file, flags):
        hfile = win32file._get_osfhandle(file.fileno())
        win32file.LockFileEx(hfile, flags, 0, 0x7fff0000, __overlapped) 

def __init__(self, filename, mode="rb"):
        self.filename = filename
        self.mode = mode
        self.file = None
        if "r" in mode:
            self.file = open_file(filename, mode)
            lock(self.file, LOCK_SH)
        elif "w" in mode or "a" in mode:
            self.file = open_file(filename, mode.replace("w", "a"))
            lock(self.file, LOCK_EX)
            if "a" not in mode:
                self.file.seek(0)
                self.file.truncate(0)
        else:
            raise RuntimeError("invalid LockedFile(...,mode)") 

def lock(file, flags):
        pass 

def lock(file, flags):
        fcntl.flock(file.fileno(), flags) 

def lock(file, flags):
        hfile = msvcrt.get_osfhandle(file.fileno())
        overlapped = OVERLAPPED()
        LockFileEx(hfile, flags, 0, 0, 0xFFFF0000, ctypes.byref(overlapped)) 

def __exit__(self, exc_type, exc_val, exc_tb):
        if self.gpu_id is not None:
            logger.info("Releasing GPU {}.".format(self.gpu_id))
        if self.lock_file is not None:
            if self._acquired_lock:
                portalocker.lock(self.lock_file, portalocker.LOCK_UN)
            self.lock_file.close()
            os.remove(self.lockfile_path) 

def __enter__(self) -> Optional[GpuDeviceType]:
        for gpu_id in self.candidates:
            lockfile_path = os.path.join(self.lock_dir, "sockeye.gpu{}.lock".format(gpu_id))
            try:
                lock_file = open(lockfile_path, 'w')
            except IOError:
                if errno.EACCES:
                    logger.warning("GPU {} is currently locked by a different process "
                                   "(Permission denied).".format(gpu_id))
                    continue
            try:
                # exclusive non-blocking lock
                portalocker.lock(lock_file, portalocker.LOCK_EX | portalocker.LOCK_NB)
                # got the lock, let's write our PID into it:
                lock_file.write("%d\n" % os.getpid())
                lock_file.flush()

                self._acquired_lock = True
                self.gpu_id = gpu_id
                self.lock_file = lock_file
                self.lockfile_path = lockfile_path

                logger.info("Acquired GPU {}.".format(gpu_id))

                return gpu_id
            except portalocker.LockException as e:
                # portalocker packages the original exception,
                # we dig it out and raise if unrelated to us
                if e.args[0].errno != errno.EAGAIN:  # pylint: disable=no-member
                    logger.error("Failed acquiring GPU lock.", exc_info=True)
                    raise e.args[0]
                else:
                    logger.debug("GPU {} is currently locked.".format(gpu_id))
        return None 

def determine_context(device_ids: List[int],
                      use_cpu: bool,
                      disable_device_locking: bool,
                      lock_dir: str,
                      exit_stack: ExitStack) -> List[mx.Context]:
    """
    Determine the MXNet context to run on (CPU or GPU).

    :param device_ids: List of device as defined from the CLI.
    :param use_cpu: Whether to use the CPU instead of GPU(s).
    :param disable_device_locking: Disable Sockeye's device locking feature.
    :param lock_dir: Directory to place device lock files in.
    :param exit_stack: An ExitStack from contextlib.

    :return: A list with the context(s) to run on.
    """
    if use_cpu:
        context = [mx.cpu()]
    else:
        num_gpus = get_num_gpus()
        check_condition(num_gpus >= 1,
                        "No GPUs found, consider running on the CPU with --use-cpu ")
        if horovod_mpi.using_horovod():
            # Running with Horovod/MPI: GPU(s) are determined by local rank
            check_condition(len(device_ids) == 1 and device_ids[0] < 0,
                            "When using Horovod, --device-ids should be a negative integer indicating the number of "
                            "GPUs each worker should use.")
            n_ids = -device_ids[0]
            context = [mx.gpu(_id + horovod_mpi.hvd.local_rank() * n_ids) for _id in range(n_ids)]
        else:
            if disable_device_locking:
                context = expand_requested_device_ids(device_ids)
            else:
                context = exit_stack.enter_context(acquire_gpus(device_ids, lock_dir=lock_dir))
            context = [mx.gpu(gpu_id) for gpu_id in context]
    return context 

def flush(self):
        """ flush():  Do nothing.

        Since a flush is issued in release(), we don't do it here. To do a flush
        here, it would be necessary to re-lock everything, and it is just easier
        and cleaner to do it all in release(), rather than requiring two lock
        ops per handle() call.

        Doing a flush() here would also introduces a window of opportunity for
        another process to write to the log file in between calling
        stream.write() and stream.flush(), which seems like a bad thing. """
        pass 

def release(self):
        """ Release file and thread locks. Flush stream and take care of closing
        stream in 'degraded' mode. """
        try:
            if self.stream.closed==False:
                self.stream.flush()
            if self._rotateFailed:
                self.stream.close()
        finally:
            try:
                unlock(self.stream_lock)
            finally:
                # release thread lock
                Handler.release(self)