Python resource.setrlimit() Examples

def test_urandom_failure(self):
        # Check urandom() failing when it is not able to open /dev/random.
        # We spawn a new process to make the test more robust (if getrlimit()
        # failed to restore the file descriptor limit after this, the whole
        # test suite would crash; this actually happened on the OS X Tiger
        # buildbot).
        code = """if 1:
            import errno
            import os
            import resource

            soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
            resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit))
            try:
                os.urandom(16)
            except OSError as e:
                assert e.errno == errno.EMFILE, e.errno
            else:
                raise AssertionError("OSError not raised")
            """
        assert_python_ok('-c', code) 

def set_ulimit():
    """Sets appropriate resource limits for the JS shell when on POSIX."""
    try:
        import resource

        # log.debug("Limit address space to 4GB")
        # This has to be twice the 2GB figure in:
        # https://hg.mozilla.org/mozilla-central/annotate/7beb238a0ea0/js/src/jit/ProcessExecutableMemory.h#l26
        # i.e. https://hg.mozilla.org/mozilla-central/rev/7beb238a0ea0 (bug 1542292) changed from 1GB to 2GB, so our
        # number here in this file has to be changed from 2GB to 4GB or else the harness (m-err.txt) will show this:
        # JS_Init failed: js::jit::InitProcessExecutableMemory() failed
        # We cannot set a limit for RLIMIT_AS for ASan binaries
        giga_byte = 2**30
        resource.setrlimit(resource.RLIMIT_AS, (4 * giga_byte, 4 * giga_byte))

        # log.debug("Limit corefiles to 0.5 GB")
        half_giga_byte = int(giga_byte // 2)
        resource.setrlimit(resource.RLIMIT_CORE, (half_giga_byte, half_giga_byte))
    except ImportError:
        # log.debug("Skipping resource import as a non-POSIX platform was detected: %s", platform.system())
        return 

def __exit__(self, *ignore_exc):
        """Restore Windows ErrorMode or core file behavior to initial value."""
        if self.old_value is None:
            return

        if sys.platform.startswith('win'):
            self._k32.SetErrorMode(self.old_value)

            if self.old_modes:
                import _testcapi
                for report_type, (old_mode, old_file) in self.old_modes.items():
                    _testcapi.CrtSetReportMode(report_type, old_mode)
                    _testcapi.CrtSetReportFile(report_type, old_file)
        else:
            import resource
            try:
                resource.setrlimit(resource.RLIMIT_CORE, self.old_value)
            except (ValueError, OSError):
                pass 

def _set(self):
        self.ulimit = {}
        for key in self.ulimit_options:
            set_value = self.params.get("vt_ulimit_%s" % key)
            if not set_value:
                continue
            # get default ulimit values in tuple (soft, hard)
            self.ulimit[key] = resource.getrlimit(self.ulimit_options[key])

            logging.info("Setting ulimit %s to %s." % (key, set_value))
            if set_value == "ulimited":
                set_value = resource.RLIM_INFINITY
            elif set_value.isdigit():
                set_value = int(set_value)
            else:
                self.test.error("%s is not supported for "
                                "setting ulimit %s" % (set_value, key))
            try:
                resource.setrlimit(self.ulimit_options[key],
                                   (set_value, set_value))
            except ValueError as error:
                self.test.error(str(error)) 

def test_urandom_failure(self):
        # Check urandom() failing when it is not able to open /dev/random.
        # We spawn a new process to make the test more robust (if getrlimit()
        # failed to restore the file descriptor limit after this, the whole
        # test suite would crash; this actually happened on the OS X Tiger
        # buildbot).
        code = """if 1:
            import errno
            import os
            import resource

            soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
            resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit))
            try:
                os.urandom(16)
            except OSError as e:
                assert e.errno == errno.EMFILE, e.errno
            else:
                raise AssertionError("OSError not raised")
            """
        assert_python_ok('-c', code) 

def __enter__(self):
        """Try to save previous ulimit, then set it to (0, 0)."""
        if resource is not None:
            try:
                self.old_limit = resource.getrlimit(resource.RLIMIT_CORE)
                resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
            except (ValueError, resource.error):
                pass

        if sys.platform == 'darwin':
            # Check if the 'Crash Reporter' on OSX was configured
            # in 'Developer' mode and warn that it will get triggered
            # when it is.
            #
            # This assumes that this context manager is used in tests
            # that might trigger the next manager.
            value = subprocess.Popen(['/usr/bin/defaults', 'read',
                    'com.apple.CrashReporter', 'DialogType'],
                    stdout=subprocess.PIPE).communicate()[0]
            if value.strip() == b'developer':
                print "this tests triggers the Crash Reporter, that is intentional"
                sys.stdout.flush() 

def prevent_core_dump():
    """ Prevent this process from generating a core dump.

        Sets the soft and hard limits for core dump size to zero. On
        Unix, this prevents the process from creating core dump
        altogether.

        """
    core_resource = resource.RLIMIT_CORE

    try:
        # Ensure the resource limit exists on this platform, by requesting
        # its current value
        resource.getrlimit(core_resource)
    except ValueError as exc:
        error = DaemonOSEnvironmentError(
            "System does not support RLIMIT_CORE resource limit (%s)" % exc)
        raise error

    # Set hard and soft limits to zero, i.e. no core dump at all
    core_limit = (0, 0)
    resource.setrlimit(core_resource, core_limit) 

def __enter__(self):
        """Try to save previous ulimit, then set it to (0, 0)."""
        if resource is not None:
            try:
                self.old_limit = resource.getrlimit(resource.RLIMIT_CORE)
                resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
            except (ValueError, resource.error):
                pass

        if sys.platform == 'darwin':
            # Check if the 'Crash Reporter' on OSX was configured
            # in 'Developer' mode and warn that it will get triggered
            # when it is.
            #
            # This assumes that this context manager is used in tests
            # that might trigger the next manager.
            value = subprocess.Popen(['/usr/bin/defaults', 'read',
                    'com.apple.CrashReporter', 'DialogType'],
                    stdout=subprocess.PIPE).communicate()[0]
            if value.strip() == b'developer':
                print "this tests triggers the Crash Reporter, that is intentional"
                sys.stdout.flush() 

def test_urandom_failure(self):
        # Check urandom() failing when it is not able to open /dev/random.
        # We spawn a new process to make the test more robust (if getrlimit()
        # failed to restore the file descriptor limit after this, the whole
        # test suite would crash; this actually happened on the OS X Tiger
        # buildbot).
        code = """if 1:
            import errno
            import os
            import resource

            soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
            resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit))
            try:
                os.urandom(16)
            except OSError as e:
                assert e.errno == errno.EMFILE, e.errno
            else:
                raise AssertionError("OSError not raised")
            """
        assert_python_ok('-c', code) 

def __enter__(self):
        """Try to save previous ulimit, then set it to (0, 0)."""
        if resource is not None:
            try:
                self.old_limit = resource.getrlimit(resource.RLIMIT_CORE)
                resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
            except (ValueError, resource.error):
                pass

        if sys.platform == 'darwin':
            # Check if the 'Crash Reporter' on OSX was configured
            # in 'Developer' mode and warn that it will get triggered
            # when it is.
            #
            # This assumes that this context manager is used in tests
            # that might trigger the next manager.
            value = subprocess.Popen(['/usr/bin/defaults', 'read',
                    'com.apple.CrashReporter', 'DialogType'],
                    stdout=subprocess.PIPE).communicate()[0]
            if value.strip() == b'developer':
                print "this tests triggers the Crash Reporter, that is intentional"
                sys.stdout.flush() 

def test_urandom_failure(self):
        # Check urandom() failing when it is not able to open /dev/random.
        # We spawn a new process to make the test more robust (if getrlimit()
        # failed to restore the file descriptor limit after this, the whole
        # test suite would crash; this actually happened on the OS X Tiger
        # buildbot).
        code = """if 1:
            import errno
            import os
            import resource

            soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
            resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit))
            try:
                os.urandom(16)
            except OSError as e:
                assert e.errno == errno.EMFILE, e.errno
            else:
                raise AssertionError("OSError not raised")
            """
        assert_python_ok('-c', code) 

def test_urandom_failure(self):
        # Check urandom() failing when it is not able to open /dev/random.
        # We spawn a new process to make the test more robust (if getrlimit()
        # failed to restore the file descriptor limit after this, the whole
        # test suite would crash; this actually happened on the OS X Tiger
        # buildbot).
        code = """if 1:
            import errno
            import os
            import resource

            soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
            resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit))
            try:
                os.urandom(16)
            except OSError as e:
                assert e.errno == errno.EMFILE, e.errno
            else:
                raise AssertionError("OSError not raised")
            """
        assert_python_ok('-c', code) 

def create_executable_limits(self) -> Optional[Callable[[], None]]:
        try:
            import resource
            from dmoj.utils.os_ext import oom_score_adj, OOM_SCORE_ADJ_MAX

            def limit_executable():
                os.setpgrp()

                # Mark compiler process as first to die in an OOM situation, just to ensure that the judge will not
                # be killed.
                try:
                    oom_score_adj(OOM_SCORE_ADJ_MAX)
                except Exception:
                    import traceback

                    traceback.print_exc()

                resource.setrlimit(resource.RLIMIT_FSIZE, (self.executable_size, self.executable_size))

            return limit_executable
        except ImportError:
            return None 

def __exit__(self, *ignore_exc):
        """Restore Windows ErrorMode or core file behavior to initial value."""
        if self.old_value is None:
            return

        if sys.platform.startswith('win'):
            self._k32.SetErrorMode(self.old_value)

            if self.old_modes:
                import msvcrt
                for report_type, (old_mode, old_file) in self.old_modes.items():
                    msvcrt.CrtSetReportMode(report_type, old_mode)
                    msvcrt.CrtSetReportFile(report_type, old_file)
        else:
            if resource is not None:
                try:
                    resource.setrlimit(resource.RLIMIT_CORE, self.old_value)
                except (ValueError, OSError):
                    pass 

def _EnforceProcessMemoryLimit(self, memory_limit):
    """Enforces a process memory limit.

    Args:
      memory_limit (int): maximum number of bytes the process is allowed
          to allocate, where 0 represents no limit and None a default of
          4 GiB.
    """
    # Resource is not supported on Windows.
    if resource:
      if memory_limit is None:
        memory_limit = 4 * 1024 * 1024 * 1024
      elif memory_limit == 0:
        memory_limit = resource.RLIM_INFINITY

      resource.setrlimit(resource.RLIMIT_DATA, (memory_limit, memory_limit)) 

def stdinpwn(self, payload):
        resource.setrlimit(resource.RLIMIT_STACK, (-1, -1))
        resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
        P = Popen(self.target, stdin=PIPE)
        print "[*] Sending buffer with lenght: " + str(len(payload))
        P.stdin.write(payload)
        while True:
            line = sys.stdin.readline()
            P.poll()
            ret = P.returncode
            if ret is None:
                P.stdin.write(line)
            else:
                if ret == -11:
                    print "[*] Child program crashed with SIGSEGV"
                else:
                    print "[-] Child program exited with code %d" % ret
                break

        print "\n If it does not work automatically, run on terminal: (cat buffer.txt ; cat) | {}".format(self.target) 

def test_collision():

    import resource
    resource.setrlimit(resource.RLIMIT_NOFILE, (102400, 102400))

    dd = {}
    ls = []
    for i in range(1 << 15):
        key = str(hashlib.sha1(str(i)).hexdigest())
        lck = key
        print 'lock is', i, lck
        l = Portlock(lck, timeout=8)
        r = l.try_lock()
        if not r:
            print 'collide', i, l.addr
            print l.socks

        dd[l.addr] = i
        ls.append(l) 

def test_operating_on_small_slices_of_lists(self):
        with self.create_executor() as fora:
            with fora.remotely:
                #1 billion integers
                a_list = range(1000000000)

                #the first million integers
                a_list_slice = a_list[:1000000]
                with helpers.python:
                    resource.setrlimit(resource.RLIMIT_AS, (2 * 1024 * 1024 * 1024, -1))

                    sum_result = sum(a_list_slice)

                a_list = None
            
            sum_result = sum_result.toLocal().result()

            self.assertEqual(sum_result, sum(range(1000000))) 

def test_above_fd_setsize(self):
        # A scalable implementation should have no problem with more than
        # FD_SETSIZE file descriptors. Since we don't know the value, we just
        # try to set the soft RLIMIT_NOFILE to the hard RLIMIT_NOFILE ceiling.
        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        try:
            resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
            self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE,
                            (soft, hard))
            NUM_FDS = min(hard, 2**16)
        except (OSError, ValueError):
            NUM_FDS = soft

        # guard for already allocated FDs (stdin, stdout...)
        NUM_FDS -= 32

        s = self.SELECTOR()
        self.addCleanup(s.close)

        for i in range(NUM_FDS // 2):
            try:
                rd, wr = self.make_socketpair()
            except OSError:
                # too many FDs, skip - note that we should only catch EMFILE
                # here, but apparently *BSD and Solaris can fail upon connect()
                # or bind() with EADDRNOTAVAIL, so let's be safe
                self.skipTest("FD limit reached")

            try:
                s.register(rd, selectors.EVENT_READ)
                s.register(wr, selectors.EVENT_WRITE)
            except OSError as e:
                if e.errno == errno.ENOSPC:
                    # this can be raised by epoll if we go over
                    # fs.epoll.max_user_watches sysctl
                    self.skipTest("FD limit reached")
                raise

        self.assertEqual(NUM_FDS // 2, len(s.select())) 

def __get_rlimit_func(self):
        def set_rlimits():
            # here we limit the logsize
            resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
            resource.setrlimit(resource.RLIMIT_FSIZE, (self.trace_log_limit, self.trace_log_limit))

        return set_rlimits 

def test_above_fd_setsize(self):
        # A scalable implementation should have no problem with more than
        # FD_SETSIZE file descriptors. Since we don't know the value, we just
        # try to set the soft RLIMIT_NOFILE to the hard RLIMIT_NOFILE ceiling.
        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        try:
            resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
            self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE,
                            (soft, hard))
            NUM_FDS = min(hard, 2**16)
        except (OSError, ValueError):
            NUM_FDS = soft

        # guard for already allocated FDs (stdin, stdout...)
        NUM_FDS -= 32

        s = self.SELECTOR()
        self.addCleanup(s.close)

        for i in range(NUM_FDS // 2):
            try:
                rd, wr = self.make_socketpair()
            except OSError:
                # too many FDs, skip - note that we should only catch EMFILE
                # here, but apparently *BSD and Solaris can fail upon connect()
                # or bind() with EADDRNOTAVAIL, so let's be safe
                self.skipTest("FD limit reached")

            try:
                s.register(rd, selectors.EVENT_READ)
                s.register(wr, selectors.EVENT_WRITE)
            except OSError as e:
                if e.errno == errno.ENOSPC:
                    # this can be raised by epoll if we go over
                    # fs.epoll.max_user_watches sysctl
                    self.skipTest("FD limit reached")
                raise

        self.assertEqual(NUM_FDS // 2, len(s.select())) 

def __find_and_set_higer_system_limit(self, res, res_name):
        resource = get_resource_lib()
        
        # first try to get the system limit, if already unlimited (-1) then we are good :)
        soft, hard = resource.getrlimit(res)
        if soft == -1 and hard == -1:
            logger.info('System resource %s is already unlimited: (soft:%s/hard:%s)' % (res_name, soft, hard))
            return
        # Ok not unlimited, maybe we can set unlimited?
        try:
            resource.setrlimit(res, (-1, -1))
            is_unlimited = True
        except ValueError:
            is_unlimited = False
        if is_unlimited:
            logger.info('System resource %s was set to unlimited' % (res_name))
            return
        # Ok maybe we cannot set unlimited, but we can try to increase it as much as we can
        can_still_increase = True
        v = hard
        if hard == -1:
            v = soft
        while can_still_increase:
            v *= 2
            try:
                logger.debug('Try to increase system limit %s to %s/%s' % (res_name, v, v))
                resource.setrlimit(res, (v, v))
            except ValueError:
                # We did find the max
                can_still_increase = False
        logger.info('System limit %s is set to maximum available: %s/%s' % (res_name, v, v)) 

def __exit__(self, *args):
        """Return core file behavior to default."""
        if self.old_limit is None:
            return
        if resource is not None:
            try:
                resource.setrlimit(resource.RLIMIT_CORE, self.old_limit)
            except (ValueError, resource.error):
                pass 

def start_fuse():
    parser = argparse.ArgumentParser(prog="GitFS")
    args = parse_args(parser)

    try:
        merge_worker, fetch_worker, router = prepare_components(args)
    except:
        return

    if args.max_open_files != -1:
        resource.setrlimit(
            resource.RLIMIT_NOFILE, (args.max_open_files, args.max_open_files)
        )

    # ready to mount it
    if sys.platform == "darwin":
        FUSE(
            router,
            args.mount_point,
            foreground=args.foreground,
            allow_root=args.allow_root,
            allow_other=args.allow_other,
            fsname=args.remote_url,
            subtype="gitfs",
        )
    else:
        FUSE(
            router,
            args.mount_point,
            foreground=args.foreground,
            nonempty=True,
            allow_root=args.allow_root,
            allow_other=args.allow_other,
            fsname=args.remote_url,
            subtype="gitfs",
        ) 

def memory_limit(percentage: float):
    if platform.system() != "Linux":
        print('Only works on linux!')
        return
    soft, hard = resource.getrlimit(resource.RLIMIT_AS)
    resource.setrlimit(resource.RLIMIT_AS, (int(get_memory() * 1024 * percentage), hard)) 

def stdinfuzz(self):
        buf = ''
        while True:
            try:
                first = True
                buf += '\x41' * self.offset
                resource.setrlimit(resource.RLIMIT_STACK, (-1, -1))
                resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
                P = Popen(self.target, stdin=PIPE)
                print "[*] Sending buffer with lenght: " + str(len(buf))
                P.stdin.write(buf + '\n')
                line = sys.stdin.readline()
                P.poll()
                ret = P.returncode

                if ret is None:
                    continue
                else:
                    if ret == -4:
                        print "\n[+] Instruction Pointer may be at: {}\n".format(str(len(buf)))
                        break
                    elif ret == -7:
                        print "\n[+] Instruction Pointer may be near: {}\n".format(str(len(buf)))
                        print "[*] Child program crashed with code: %d\n" % ret
                        continue
                    elif ret == -11:
                        print "[*] Child program crashed with SIGSEGV code: %d\n" % ret
                        print "\n[*] Hit enter to continue.\n"
                        continue
                    else:
                        print "[*] Child program exited with code: %d\n" % ret
                        print "\n[*] Hit enter to continue.\n"
                        continue


            except KeyboardInterrupt:
                break 

def test_above_fd_setsize(self):
        # A scalable implementation should have no problem with more than
        # FD_SETSIZE file descriptors. Since we don't know the value, we just
        # try to set the soft RLIMIT_NOFILE to the hard RLIMIT_NOFILE ceiling.
        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        try:
            resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
            self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE,
                            (soft, hard))
            NUM_FDS = min(hard, 2**16)
        except (OSError, ValueError):
            NUM_FDS = soft

        # guard for already allocated FDs (stdin, stdout...)
        NUM_FDS -= 32

        s = self.SELECTOR()
        self.addCleanup(s.close)

        for i in range(NUM_FDS // 2):
            try:
                rd, wr = self.make_socketpair()
            except OSError:
                # too many FDs, skip - note that we should only catch EMFILE
                # here, but apparently *BSD and Solaris can fail upon connect()
                # or bind() with EADDRNOTAVAIL, so let's be safe
                self.skipTest("FD limit reached")

            try:
                s.register(rd, selectors.EVENT_READ)
                s.register(wr, selectors.EVENT_WRITE)
            except OSError as e:
                if e.errno == errno.ENOSPC:
                    # this can be raised by epoll if we go over
                    # fs.epoll.max_user_watches sysctl
                    self.skipTest("FD limit reached")
                raise

        self.assertEqual(NUM_FDS // 2, len(s.select())) 

def test_file_management(self):
        tempDir = tempfile.mkdtemp()

        curOpenFiles = len(os.listdir('/proc/%s/fd' % os.getpid()))

        OPEN_FILE_LIMIT = 200

        if curOpenFiles >= OPEN_FILE_LIMIT:
            os.system("ls -alh /proc/%s/fd" % os.getpid())

        self.assertTrue(curOpenFiles < OPEN_FILE_LIMIT, "Too many open files: %s" % curOpenFiles)

        soft, hard = resource.getrlimit(resource.RLIMIT_OFILE)

        harness = self.getHarness(inMemory=True,
                                  cachePathOverride=tempDir,
                                  maxOpenFiles=15)
        try:
            v1 = harness.newView()
            resource.setrlimit(resource.RLIMIT_OFILE, (curOpenFiles + 30, hard))

            for ix in range(128):
                space = SharedState.Keyspace("TakeHighestIdKeyType",
                                             NativeJson.Json("TestSpace%s" % ix),
                                             1)
                rng = SharedState.KeyRange(space, 0, None, None, True, False)
                v1.subscribe(rng)
                key = SharedState.Key(space, (NativeJson.Json('key%s' % ix),))
                with SharedState.Transaction(v1):
                    v1[key] = NativeJson.Json('value %s' % ix)
        finally:
            time.sleep(0.01)
            harness.teardown()
            resource.setrlimit(resource.RLIMIT_OFILE, (soft, hard))
            try:
                shutil.rmtree(tempDir)
            except:
                pass 

def preexec_fn():
    resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) 

def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
        assert self.previous_pattern is not None
        with open(HANDLER_PATH, "w") as f:
            f.write(self.previous_pattern)
        if self.old_core_rlimit is not None:
            resource.setrlimit(resource.RLIMIT_CORE, self.old_core_rlimit)
        if self.handler_script is not None:
            os.unlink(self.handler_script.name)