Python oslo_concurrency.processutils.get_worker_count() Examples

The following are 25 code examples of oslo_concurrency.processutils.get_worker_count(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module oslo_concurrency.processutils , or try the search function .
Example #1
Source File: service.py    From zun with Apache License 2.0 6 votes vote down vote up
def __init__(self, name, use_ssl=False):
        """Initialize, but do not start the WSGI server.

        :param name: The name of the WSGI server given to the loader.
        :param use_ssl: Wraps the socket in an SSL context if True.
        :returns: None
        """
        self.name = name
        self.app = app.load_app()
        self.workers = (CONF.api.workers or processutils.get_worker_count())
        if self.workers and self.workers < 1:
            raise exception.ConfigInvalid(
                _("api_workers value of %d is invalid, "
                  "must be greater than 0.") % self.workers)

        self.server = wsgi.Server(CONF, name, self.app,
                                  host=CONF.api.host_ip,
                                  port=CONF.api.port,
                                  use_ssl=use_ssl) 
Example #2
Source File: service.py    From vdi-broker with Apache License 2.0 6 votes vote down vote up
def __init__(self, name):
        self._host = CONF.api_listen
        self._port = CONF.api_listen_port

        if platform.system() == "Windows":
            self._workers = 1
        else:
            self._workers = (
                CONF.api_workers or processutils.get_worker_count())

        self._loader = wsgi.Loader(CONF)
        self._app = self._loader.load_app(name)

        self._server = wsgi.Server(CONF,
                                   name,
                                   self._app,
                                   host=self._host,
                                   port=self._port) 
Example #3
Source File: test_api.py    From magnum with Apache License 2.0 6 votes vote down vote up
def test_api_https(self, mock_prep, mock_app, mock_run,
                       mock_exist, mock_base):
        self.config(enabled_ssl=True,
                    ssl_cert_file='tmp_crt',
                    ssl_key_file='tmp_key',
                    group='api')
        mock_exist.side_effect = [True, True]

        api.main()

        app = mock_app.load_app.return_value
        mock_prep.assert_called_once_with(mock.ANY)
        mock_app.load_app.assert_called_once_with()
        mock_exist.assert_has_calls([mock.call('tmp_crt'),
                                     mock.call('tmp_key')])
        workers = processutils.get_worker_count()
        mock_run.assert_called_once_with(base.CONF.api.host,
                                         base.CONF.api.port, app,
                                         processes=workers,
                                         ssl_context=('tmp_crt', 'tmp_key')) 
Example #4
Source File: image_uploader.py    From tripleo-common with Apache License 2.0 6 votes vote down vote up
def _get_executor(self):
        """Get executor type based on lock object

        We check to see if the lock object is not set or if it is a threading
        lock. We cannot check if it is a ProcessLock due to the side effect
        of trying to include ProcessLock when running under Mistral breaks
        Mistral.
        """
        if not self.lock or isinstance(self.lock, threadinglock.ThreadingLock):
            # workers will scale from 2 to 8 based on the cpu count // 2
            workers = min(max(2, processutils.get_worker_count() // 2), 8)
            return futures.ThreadPoolExecutor(max_workers=workers)
        else:
            # there really isn't an improvement with > 4 workers due to the
            # container layer overlaps. The higher the workers, the more
            # RAM required which can lead to OOMs. It's best to limit to 4
            return futures.ProcessPoolExecutor(max_workers=4) 
Example #5
Source File: image_uploader.py    From tripleo-common with Apache License 2.0 6 votes vote down vote up
def run_tasks(self):
        if not self.upload_tasks:
            return
        local_images = []

        # Pull a single image first, to avoid duplicate pulls of the
        # same base layers
        local_images.extend(upload_task(args=self.upload_tasks.pop()))

        # workers will be half the CPU count, to a minimum of 2
        workers = max(2, (processutils.get_worker_count() - 1))
        with futures.ThreadPoolExecutor(max_workers=workers) as p:
            for result in p.map(upload_task, self.upload_tasks):
                local_images.extend(result)
        LOG.info('result %s' % local_images)

        # Do cleanup after all the uploads so common layers don't get deleted
        # repeatedly
        self.cleanup(local_images) 
Example #6
Source File: service.py    From qinling with Apache License 2.0 6 votes vote down vote up
def __init__(self):
        self.app = app.setup_app()

        self.workers = CONF.api.api_workers
        if self.workers is not None and self.workers < 1:
            LOG.warning(
                "Value of config option api_workers must be integer "
                "greater than 1.  Input value ignored."
            )
            self.workers = None
        self.workers = self.workers or processutils.get_worker_count()

        self.server = wsgi.Server(
            cfg.CONF,
            "qinling_api",
            self.app,
            host=cfg.CONF.api.host,
            port=cfg.CONF.api.port,
            use_ssl=cfg.CONF.api.enable_ssl_api
        ) 
Example #7
Source File: service.py    From cyborg with Apache License 2.0 6 votes vote down vote up
def __init__(self, name, use_ssl=False):
        """Initialize, but do not start the WSGI server.

        :param name: The name of the WSGI server given to the loader.
        :param use_ssl: Wraps the socket in an SSL context if True.
        :returns: None
        """
        self.name = name
        self.app = app.load_app()
        self.workers = (CONF.api.api_workers or
                        processutils.get_worker_count())
        if self.workers and self.workers < 1:
            raise exception.ConfigInvalid(
                _("api_workers value of %d is invalid, "
                  "must be greater than 0.") % self.workers)

        self.server = wsgi.Server(CONF, self.name, self.app,
                                  host=CONF.api.host_ip,
                                  port=CONF.api.port,
                                  use_ssl=use_ssl) 
Example #8
Source File: service.py    From coriolis with GNU Affero General Public License v3.0 6 votes vote down vote up
def __init__(self, name):
        self._host = CONF.api_migration_listen
        self._port = CONF.api_migration_listen_port

        if platform.system() == "Windows":
            self._workers = 1
        else:
            self._workers = (
                CONF.api_migration_workers or processutils.get_worker_count())

        self._loader = wsgi.Loader(CONF)
        self._app = self._loader.load_app(name)

        self._server = wsgi.Server(CONF,
                                   name,
                                   self._app,
                                   host=self._host,
                                   port=self._port) 
Example #9
Source File: default.py    From watcher with Apache License 2.0 6 votes vote down vote up
def get_config_opts(cls):
        return [
            cfg.IntOpt(
                'max_workers',
                default=processutils.get_worker_count(),
                min=1,
                required=True,
                help='Number of workers for taskflow engine '
                     'to execute actions.'),
            cfg.DictOpt(
                'action_execution_rule',
                default={},
                help='The execution rule for linked actions,'
                     'the key is strategy name and '
                     'value ALWAYS means all actions will be executed,'
                     'value ANY means if previous action executes '
                     'success, the next action will be ignored.'
                     'None means ALWAYS.')
            ] 
Example #10
Source File: service.py    From karbor with Apache License 2.0 5 votes vote down vote up
def __init__(self, name, loader=None):
        """Initialize, but do not start the WSGI server.

        :param name: The name of the WSGI server given to the loader.
        :param loader: Loads the WSGI application using the given name.
        :returns: None

        """
        self.name = name
        self.manager = self._get_manager()
        self.loader = loader or wsgi.Loader(CONF)
        self.app = self.loader.load_app(name)
        self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0")
        self.port = getattr(CONF, '%s_listen_port' % name, 0)
        self.workers = (getattr(CONF, '%s_workers' % name, None) or
                        processutils.get_worker_count())
        if self.workers and self.workers < 1:
            worker_name = '%s_workers' % name
            msg = (_("%(worker_name)s value of %(workers)d is invalid, "
                     "must be greater than 0.") %
                   {'worker_name': worker_name,
                    'workers': self.workers})
            raise exception.InvalidInput(msg)

        self.server = wsgi.Server(CONF,
                                  name,
                                  self.app,
                                  host=self.host,
                                  port=self.port)
        super(WSGIService, self).__init__() 
Example #11
Source File: test_service.py    From karbor with Apache License 2.0 5 votes vote down vote up
def test_workers_set_zero_user_setting(self, mock_loader):
        self.override_config('osapi_karbor_listen_port',
                             CONF.test_service_listen_port)
        self.override_config('osapi_karbor_workers', 0)
        test_service = service.WSGIService("osapi_karbor")
        # If a value less than 1 is used, defaults to number of procs
        # available
        self.assertEqual(processutils.get_worker_count(),
                         test_service.workers)
        self.assertTrue(mock_loader.called) 
Example #12
Source File: test_service.py    From karbor with Apache License 2.0 5 votes vote down vote up
def test_workers_set_default(self, mock_loader):
        self.override_config('osapi_karbor_listen_port',
                             CONF.test_service_listen_port)
        test_service = service.WSGIService("osapi_karbor")
        self.assertEqual(processutils.get_worker_count(),
                         test_service.workers)
        self.assertTrue(mock_loader.called) 
Example #13
Source File: service.py    From coriolis with GNU Affero General Public License v3.0 5 votes vote down vote up
def __init__(self, topic, endpoints, version, worker_count=None):
        target = messaging.Target(topic=topic,
                                  server=utils.get_hostname(),
                                  version=version)
        self._server = rpc.get_server(target, endpoints)

        self._workers = (worker_count or CONF.messaging_workers or
                         processutils.get_worker_count()) 
Example #14
Source File: test_processutils.py    From oslo.concurrency with Apache License 2.0 5 votes vote down vote up
def test_get_worker_count_cpu_count_not_implemented(self,
                                                        mock_cpu_count):
        self.assertEqual(1, processutils.get_worker_count()) 
Example #15
Source File: test_processutils.py    From oslo.concurrency with Apache License 2.0 5 votes vote down vote up
def test_get_worker_count(self, mock_cpu_count):
        self.assertEqual(8, processutils.get_worker_count()) 
Example #16
Source File: service.py    From watcher with Apache License 2.0 5 votes vote down vote up
def __init__(self, service_name, use_ssl=False):
        """Initialize, but do not start the WSGI server.

        :param service_name: The service name of the WSGI server.
        :param use_ssl: Wraps the socket in an SSL context if True.
        """
        self.service_name = service_name
        self.app = app.VersionSelectorApplication()
        self.workers = (CONF.api.workers or
                        processutils.get_worker_count())
        self.server = wsgi.Server(CONF, self.service_name, self.app,
                                  host=CONF.api.host,
                                  port=CONF.api.port,
                                  use_ssl=use_ssl,
                                  logger_name=self.service_name) 
Example #17
Source File: service.py    From masakari with Apache License 2.0 5 votes vote down vote up
def __init__(self, name, loader=None, use_ssl=False, max_url_len=None):
        """Initialize, but do not start the WSGI server.

        :param name: The name of the WSGI server given to the loader.
        :param loader: Loads the WSGI application using the given name.
        :returns: None
        """
        self.name = name
        self.binary = 'masakari-%s' % name
        self.topic = None
        self.loader = loader or wsgi.Loader()
        self.app = self.loader.load_app(name)
        self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0")
        self.port = getattr(CONF, '%s_listen_port' % name, 0)

        self.workers = (getattr(CONF, '%s_workers' % name, None) or
                        processutils.get_worker_count())

        if self.workers and self.workers < 1:
            worker_name = '%s_workers' % name
            msg = (_("%(worker_name)s value of %(workers)s is invalid, "
                     "must be greater than 0") %
                   {'worker_name': worker_name,
                    'workers': str(self.workers)})
            raise exception.InvalidInput(msg)

        self.use_ssl = use_ssl
        self.server = wsgi.Server(name,
                                  self.app,
                                  host=self.host,
                                  port=self.port,
                                  use_ssl=self.use_ssl,
                                  max_url_len=max_url_len) 
Example #18
Source File: test_service.py    From masakari with Apache License 2.0 5 votes vote down vote up
def test_workers_set_zero_user_setting(self):
        CONF.set_override('masakari_api_workers', 0)
        test_service = service.WSGIService("masakari_api")
        # If a value less than 1 is used, defaults to number of procs available
        self.assertEqual(test_service.workers, processutils.get_worker_count()) 
Example #19
Source File: test_service.py    From masakari with Apache License 2.0 5 votes vote down vote up
def test_workers_set_default(self):
        test_service = service.WSGIService("masakari_api")
        self.assertEqual(test_service.workers, processutils.get_worker_count()) 
Example #20
Source File: api.py    From magnum with Apache License 2.0 5 votes vote down vote up
def main():
    service.prepare_service(sys.argv)

    gmr.TextGuruMeditation.setup_autorun(version)

    # Enable object backporting via the conductor
    base.MagnumObject.indirection_api = base.MagnumObjectIndirectionAPI()

    app = api_app.load_app()

    # Setup OSprofiler for WSGI service
    profiler.setup('magnum-api', CONF.host)

    # SSL configuration
    use_ssl = CONF.api.enabled_ssl

    # Create the WSGI server and start it
    host, port = CONF.api.host, CONF.api.port

    LOG.info('Starting server in PID %s', os.getpid())
    LOG.debug("Configuration:")
    CONF.log_opt_values(LOG, logging.DEBUG)

    LOG.info('Serving on %(proto)s://%(host)s:%(port)s',
             dict(proto="https" if use_ssl else "http", host=host, port=port))

    workers = CONF.api.workers
    if not workers:
        workers = processutils.get_worker_count()
    LOG.info('Server will handle each request in a new process up to'
             ' %s concurrent processes', workers)
    serving.run_simple(host, port, app, processes=workers,
                       ssl_context=_get_ssl_configs(use_ssl)) 
Example #21
Source File: conductor.py    From magnum with Apache License 2.0 5 votes vote down vote up
def main():
    magnum_service.prepare_service(sys.argv)

    gmr.TextGuruMeditation.setup_autorun(version)

    LOG.info('Starting server in PID %s', os.getpid())
    LOG.debug("Configuration:")
    CONF.log_opt_values(LOG, logging.DEBUG)

    conductor_id = short_id.generate_id()
    endpoints = [
        indirection_api.Handler(),
        cluster_conductor.Handler(),
        conductor_listener.Handler(),
        ca_conductor.Handler(),
        federation_conductor.Handler(),
        nodegroup_conductor.Handler(),
    ]

    server = rpc_service.Service.create(CONF.conductor.topic,
                                        conductor_id, endpoints,
                                        binary='magnum-conductor')
    workers = CONF.conductor.workers
    if not workers:
        workers = processutils.get_worker_count()
    launcher = service.launch(CONF, server, workers=workers)

    # NOTE(mnaser): We create the periodic tasks here so that they
    #               can be attached to the main process and not
    #               duplicated in all the children if multiple
    #               workers are being used.
    server.create_periodic_tasks()
    server.start()

    launcher.wait() 
Example #22
Source File: test_conductor.py    From magnum with Apache License 2.0 5 votes vote down vote up
def test_conductor(self, mock_prep, mock_rpc, mock_launch):
        conductor.main()

        server = mock_rpc.Service.create.return_value
        launcher = mock_launch.return_value
        mock_prep.assert_called_once_with(mock.ANY)
        mock_rpc.Service.create.assert_called_once_with(
            base.CONF.conductor.topic,
            mock.ANY, mock.ANY, binary='magnum-conductor')
        workers = processutils.get_worker_count()
        mock_launch.assert_called_once_with(base.CONF, server,
                                            workers=workers)
        launcher.wait.assert_called_once_with() 
Example #23
Source File: test_api.py    From magnum with Apache License 2.0 5 votes vote down vote up
def test_api_http(self, mock_prep, mock_app, mock_run, mock_base):
        api.main()

        app = mock_app.load_app.return_value
        mock_prep.assert_called_once_with(mock.ANY)
        mock_app.load_app.assert_called_once_with()
        workers = processutils.get_worker_count()
        mock_run.assert_called_once_with(base.CONF.api.host,
                                         base.CONF.api.port,
                                         app, processes=workers,
                                         ssl_context=None) 
Example #24
Source File: service.py    From vdi-broker with Apache License 2.0 5 votes vote down vote up
def __init__(self, topic, endpoints, version):
        target = messaging.Target(topic=topic,
                                  server=utils.get_hostname(),
                                  version=version)
        self._server = rpc.get_server(target, endpoints)

        self._workers = (CONF.messaging_workers or
                         processutils.get_worker_count()) 
Example #25
Source File: service.py    From ec2-api with Apache License 2.0 5 votes vote down vote up
def __init__(self, name, loader=None, max_url_len=None):
        """Initialize, but do not start the WSGI server.

        :param name: The name of the WSGI server given to the loader.
        :param loader: Loads the WSGI application using the given name.
        :returns: None

        """
        self.name = name
        self.manager = self._get_manager()
        self.loader = loader or wsgi.Loader()
        self.app = self.loader.load_app(name)
        self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0")
        self.port = getattr(CONF, '%s_listen_port' % name, 0)
        self.use_ssl = getattr(CONF, '%s_use_ssl' % name, False)
        self.workers = (getattr(CONF, '%s_workers' % name, None) or
                        processutils.get_worker_count())
        if self.workers and self.workers < 1:
            worker_name = '%s_workers' % name
            msg = (_("%(worker_name)s value of %(workers)s is invalid, "
                     "must be greater than 0") %
                   {'worker_name': worker_name,
                    'workers': str(self.workers)})
            raise exception.InvalidInput(msg)
        self.server = wsgi.Server(name,
                                  self.app,
                                  host=self.host,
                                  port=self.port,
                                  use_ssl=self.use_ssl,
                                  max_url_len=max_url_len)
        # Pull back actual port used
        self.port = self.server.port