Python fabric.api.env.roledefs() Examples

def chef_query(query, api=None, hostname_attr=DEFAULT_HOSTNAME_ATTR, environment=_default_environment):
    """A decorator to use an arbitrary Chef search query to find nodes to execute on.

    This is used like Fabric's ``roles()`` decorator, but accepts a Chef search query.

    Example::

        from chef.fabric import chef_query

        @chef_query('roles:web AND tags:active')
        @task
        def deploy():
            pass

    .. versionadded:: 0.2.1
    """
    api = _api(api)
    if api.version_parsed < Environment.api_version_parsed and environment is not None:
        raise ChefAPIVersionError('Environment support requires Chef API 0.10 or greater')
    rolename = 'query_'+query
    env.setdefault('roledefs', {})[rolename] = Roledef(query, api, hostname_attr, environment)
    return lambda fn: roles(rolename)(fn) 

def test_collect_query_info(self, requests_url_mock, requests_get_mock,
                                mkdir_mock, open_mock,
                                req_json_mock, json_dumps_mock):
        query_id = "1234_abcd"
        query_info_file_name = path.join(TMP_PRESTO_DEBUG,
                                         "query_info_" + query_id + ".json")
        file_obj = open_mock.return_value.__enter__.return_value
        requests_get_mock.return_value.json.return_value = req_json_mock
        requests_get_mock.return_value.status_code = requests.codes.ok
        env.host = "myhost"
        env.roledefs["coordinator"] = ["myhost"]

        collect.query_info(query_id)

        mkdir_mock.assert_called_with(TMP_PRESTO_DEBUG)

        open_mock.assert_called_with(query_info_file_name, "w")

        json_dumps_mock.assert_called_with(req_json_mock, indent=4)

        file_obj.write.assert_called_with(json_dumps_mock.return_value) 

def validate(conf):
        validate_presto_conf(conf)
        if 'coordinator' not in conf['config.properties']:
            raise ConfigurationError('Must specify coordinator=false in '
                                     'worker\'s config.properties')
        if conf['config.properties']['coordinator'] != 'false':
            raise ConfigurationError('Coordinator must be false in the '
                                     'worker\'s config.properties')
        uri = urlparse.urlparse(conf['config.properties']['discovery.uri'])
        if Worker.is_localhost(uri.hostname) and len(env.roledefs['all']) > 1:
            raise ConfigurationError(
                'discovery.uri should not be localhost in a '
                'multi-node cluster, but found ' + urlparse.urlunparse(uri) +
                '.  You may have encountered this error by '
                'choosing a coordinator that is localhost and a worker that '
                'is not.  The default discovery-uri is '
                'http://<coordinator>:8080')
        return conf 

def test_collect_node_information(self, mock_ext_ip, mock_service,
                                      mock_version, mock_presto_config):
        env.roledefs = {
            'coordinator': ['Node1'],
            'all': ['Node1']
        }
        mock_ext_ip.side_effect = ['IP1', 'IP3', 'IP4']
        mock_service.side_effect = [True, False, Exception('Not running')]
        mock_version.side_effect = ['', 'Presto not installed', '', '']

        self.assertEqual(('IP1', True, ''), server.collect_node_information())
        self.assertEqual(('Unknown', False, 'Presto not installed'),
                         server.collect_node_information())
        self.assertEqual(('IP3', False, ''), server.collect_node_information())
        self.assertEqual(('IP4', False, ''),
                         server.collect_node_information()) 

def test_validate_default(self):
        env.roledefs['coordinator'] = 'localhost'
        env.roledefs['workers'] = ['localhost']
        conf = coordinator.Coordinator().build_all_defaults()
        self.assertEqual(conf, coordinator.Coordinator.validate(conf)) 

def test_by_role_coordinator(self):
        env.roledefs = self.TEST_ROLEDEFS

        callback = Mock()

        fabricapi.by_role_coordinator('worker0', callback)
        self.assertFalse(callback.called, 'coordinator callback called for ' +
                         'worker')
        fabricapi.by_role_coordinator('coordinator', callback)
        callback.assert_any_call() 

def test_by_role_worker(self):
        env.roledefs = self.TEST_ROLEDEFS

        callback = Mock()

        fabricapi.by_role_worker('coordinator', callback)
        self.assertFalse(callback.called, 'worker callback called for ' +
                         'coordinator')
        fabricapi.by_role_worker('worker0', callback)
        callback.assert_any_call() 

def assert_is_worker(self, roledefs):
        def check(*args, **kwargs):
            self.assertTrue(env.host in roledefs.get('worker'))
        return check 

def assert_is_coordinator(self, roledefs):
        def check(*args, **kwargs):
            self.assertTrue(env.host in roledefs.get('coordinator'))
        return check 

def test_by_rolename_coordinator(self):
        callback = Mock()
        callback.side_effect = self.assert_is_coordinator(self.TEST_ROLEDEFS)
        env.roledefs = self.TEST_ROLEDEFS

        env.host = 'worker0'
        fabricapi.by_rolename(env.host, 'coordinator', callback)
        self.assertFalse(callback.called)

        env.host = 'coordinator'
        fabricapi.by_rolename(env.host, 'coordinator', callback)
        self.assertTrue(callback.called) 

def test_by_rolename_all(self):
        callback = Mock()
        env.roledefs = self.TEST_ROLEDEFS

        env.host = 'worker0'
        fabricapi.by_rolename(env.host, None, callback)
        self.assertTrue(callback.called)

        callback.reset_mock()

        env.host = 'coordinator'
        fabricapi.by_rolename(env.host, None, callback)
        self.assertTrue(callback.called) 

def test_build_all_defaults(self):
        env.roledefs['coordinator'] = 'a'
        env.roledefs['workers'] = ['b', 'c']
        actual_default = coordinator.Coordinator().build_all_defaults()
        expected = {'node.properties':
                    {'node.environment': 'presto',
                     'node.data-dir': '/var/lib/presto/data',
                     'node.launcher-log-file': '/var/log/presto/launcher.log',
                     'node.server-log-file': '/var/log/presto/server.log',
                     'catalog.config-dir': '/etc/presto/catalog',
                     'plugin.dir': '/usr/lib/presto/lib/plugin'},
                    'jvm.config': ['-server',
                                   '-Xmx16G',
                                   '-XX:-UseBiasedLocking',
                                   '-XX:+UseG1GC',
                                   '-XX:G1HeapRegionSize=32M',
                                   '-XX:+ExplicitGCInvokesConcurrent',
                                   '-XX:+HeapDumpOnOutOfMemoryError',
                                   '-XX:+UseGCOverheadLimit',
                                   '-XX:+ExitOnOutOfMemoryError',
                                   '-XX:ReservedCodeCacheSize=512M',
                                   '-DHADOOP_USER_NAME=hive'],
                    'config.properties': {
                        'coordinator': 'true',
                        'discovery-server.enabled': 'true',
                        'discovery.uri': 'http://a:8080',
                        'http-server.http.port': '8080',
                        'node-scheduler.include-coordinator': 'false',
                        'query.max-memory': '50GB',
                        'query.max-memory-per-node': '8GB'}
                    }

        self.assertEqual(actual_default, expected) 

def test_defaults_coord_is_worker(self):
        env.roledefs['coordinator'] = ['a']
        env.roledefs['worker'] = ['a', 'b', 'c']
        actual_default = coordinator.Coordinator().build_all_defaults()
        expected = {'node.properties': {
                    'node.environment': 'presto',
                    'node.data-dir': '/var/lib/presto/data',
                    'node.launcher-log-file': '/var/log/presto/launcher.log',
                    'node.server-log-file': '/var/log/presto/server.log',
                    'catalog.config-dir': '/etc/presto/catalog',
                    'plugin.dir': '/usr/lib/presto/lib/plugin'},
                    'jvm.config': ['-server',
                                   '-Xmx16G',
                                   '-XX:-UseBiasedLocking',
                                   '-XX:+UseG1GC',
                                   '-XX:G1HeapRegionSize=32M',
                                   '-XX:+ExplicitGCInvokesConcurrent',
                                   '-XX:+HeapDumpOnOutOfMemoryError',
                                   '-XX:+UseGCOverheadLimit',
                                   '-XX:+ExitOnOutOfMemoryError',
                                   '-XX:ReservedCodeCacheSize=512M',
                                   '-DHADOOP_USER_NAME=hive'],
                    'config.properties': {
                        'coordinator': 'true',
                        'discovery-server.enabled': 'true',
                        'discovery.uri': 'http://a:8080',
                        'http-server.http.port': '8080',
                        'node-scheduler.include-coordinator': 'true',
                        'query.max-memory': '50GB',
                        'query.max-memory-per-node': '8GB'}
                    }

        self.assertEqual(actual_default, expected) 

def test_get_conf(self, get_presto_conf_mock, write_mock):
        env.roledefs['coordinator'] = ['j']
        file_conf = {'node.properties': {'my-property': 'value',
                                         'node.environment': 'test'}}
        get_presto_conf_mock.return_value = file_conf
        expected = {'node.properties':
                    {'my-property': 'value',
                     'node.environment': 'test'},
                    'jvm.config': ['-server',
                                   '-Xmx16G',
                                   '-XX:-UseBiasedLocking',
                                   '-XX:+UseG1GC',
                                   '-XX:G1HeapRegionSize=32M',
                                   '-XX:+ExplicitGCInvokesConcurrent',
                                   '-XX:+HeapDumpOnOutOfMemoryError',
                                   '-XX:+UseGCOverheadLimit',
                                   '-XX:+ExitOnOutOfMemoryError',
                                   '-XX:ReservedCodeCacheSize=512M',
                                   '-DHADOOP_USER_NAME=hive'],
                    'config.properties': {'coordinator': 'false',
                                          'discovery.uri': 'http://j:8080',
                                          'http-server.http.port': '8080',
                                          'query.max-memory': '50GB',
                                          'query.max-memory-per-node': '8GB'}
                    }
        self.assertEqual(workers.Worker().get_conf(), expected) 

def test_get_conf(self, get_conf_from_file_mock, write_mock):
        env.roledefs['coordinator'] = 'j'
        env.roledefs['workers'] = ['K', 'L']
        file_conf = {'node.properties': {'my-property': 'value',
                                         'node.environment': 'test'}}
        get_conf_from_file_mock.return_value = file_conf
        expected = {'node.properties':
                    {'my-property': 'value',
                     'node.environment': 'test'},
                    'jvm.config': ['-server',
                                   '-Xmx16G',
                                   '-XX:-UseBiasedLocking',
                                   '-XX:+UseG1GC',
                                   '-XX:G1HeapRegionSize=32M',
                                   '-XX:+ExplicitGCInvokesConcurrent',
                                   '-XX:+HeapDumpOnOutOfMemoryError',
                                   '-XX:+UseGCOverheadLimit',
                                   '-XX:+ExitOnOutOfMemoryError',
                                   '-XX:ReservedCodeCacheSize=512M',
                                   '-DHADOOP_USER_NAME=hive'],
                    'config.properties': {
                        'coordinator': 'true',
                        'discovery-server.enabled': 'true',
                        'discovery.uri': 'http://j:8080',
                        'http-server.http.port': '8080',
                        'node-scheduler.include-coordinator': 'false',
                        'query.max-memory': '50GB',
                        'query.max-memory-per-node': '8GB'}
                    }

        self.assertEqual(coordinator.Coordinator().get_conf(), expected) 

def test_query_info_not_run_on_workers(self, req_get_mock):
        env.host = ["worker1"]
        env.roledefs["worker"] = ["worker1"]
        collect.query_info("any_query_id")
        assert not req_get_mock.called 

def test_query_info_fail_invalid_id(self, req_get_mock, requests_url):
        env.host = "myhost"
        env.roledefs["coordinator"] = ["myhost"]
        query_id = "invalid_id"
        req_get_mock.return_value.status_code = requests.codes.ok + 10
        self.assertRaisesRegexp(SystemExit, "Unable to retrieve information. "
                                            "Please check that the query_id "
                                            "is correct, or check that server "
                                            "is up with command: "
                                            "server status",
                                collect.query_info, query_id) 

def test_collect_system_info(self, requests_url_mock, requests_get_mock,
                                 makedirs_mock, open_mock,
                                 execute_mock, req_json_mock,
                                 json_dumps_mock, catalog_info_mock,
                                 make_tarfile_mock, mock_presto_config):
        downloaded_sys_info_loc = path.join(TMP_PRESTO_DEBUG, "sysinfo")
        node_info_file_name = path.join(downloaded_sys_info_loc,
                                        "node_info.json")
        conn_info_file_name = path.join(downloaded_sys_info_loc,
                                        "catalog_info.txt")

        file_obj = open_mock.return_value.__enter__.return_value
        requests_get_mock.return_value.json.return_value = req_json_mock
        requests_get_mock.return_value.status_code = requests.codes.ok
        catalog_info = catalog_info_mock.return_value

        env.host = "myhost"
        env.roledefs["coordinator"] = ["myhost"]
        collect.system_info()

        makedirs_mock.assert_called_with(downloaded_sys_info_loc)
        makedirs_mock.assert_called_with(downloaded_sys_info_loc)

        open_mock.assert_any_call(node_info_file_name, "w")

        json_dumps_mock.assert_called_with(req_json_mock, indent=4)

        file_obj.write.assert_any_call(json_dumps_mock.return_value)

        open_mock.assert_any_call(conn_info_file_name, "w")

        assert catalog_info_mock.called

        file_obj.write.assert_any_call(catalog_info + '\n')

        execute_mock.assert_called_with(collect.get_system_info,
                                        downloaded_sys_info_loc, roles=[])

        make_tarfile_mock.assert_called_with(OUTPUT_FILENAME_FOR_SYS_INFO,
                                             downloaded_sys_info_loc) 

def get_current_roles():
    """
    Determines the list of roles, that the current host is assigned to. If ``env.roledefs`` is not set, an empty list
    is returned.

    :return: List of roles of the current host.
    :rtype: list
    """
    current_host = env.host_string
    roledefs = env.get('roledefs')
    if roledefs:
        return [role for role, hosts in six.iteritems(roledefs) if current_host in hosts]
    return [] 

def get_role_addresses(role_name, interface_name):
    roledefs = env.get('roledefs')
    clients = env.get('docker_clients')
    if roledefs and clients:
        role_hosts = roledefs.get(role_name)
        if role_hosts:
            return set(client_config.interfaces[interface_name]
                       for client_name, client_config in six.iteritems(clients)
                       if client_config.get('fabric_host') in role_hosts)
    return set() 

def test_check_success_status(self, mock_run_sql, string_config_mock, mock_run, mock_presto_config):
        env.roledefs = {
            'coordinator': ['Node1'],
            'worker': ['Node1', 'Node2', 'Node3', 'Node4'],
            'all': ['Node1', 'Node2', 'Node3', 'Node4']
        }
        env.hosts = env.roledefs['all']
        env.host = 'Node1'
        string_config_mock.return_value = 'Node1'
        mock_run_sql.return_value = [['Node2', 'some stuff'], ['Node1', 'some other stuff']]
        self.assertEqual(server.check_server_status(), True) 

def chef_environment(name, api=None):
    """A Fabric task to set the current Chef environment context.

    This task works alongside :func:`~chef.fabric.chef_roledefs` to set the
    Chef environment to be used in future role queries.

    Example::

        from chef.fabric import chef_environment, chef_roledefs
        env.roledefs = chef_roledefs()

    .. code-block:: bash

        $ fab env:production deploy

    The task can be configured slightly via Fabric ``env`` values.

    ``env.chef_environment_task_alias`` sets the task alias, defaulting to "env".
    This value must be set **before** :mod:`chef.fabric` is imported.

    ``env.chef_environment_validate`` sets if :class:`~chef.Environment` names
    should be validated before use. Defaults to True.

    .. versionadded:: 0.2
    """
    if env.get('chef_environment_validate', True):
        api = _api(api)
        chef_env = Environment(name, api=api)
        if not chef_env.exists:
            raise ChefError('Unknown Chef environment: %s' % name)
    env['chef_environment'] = name 

def decommissioned_host(host):
    more_than_one_webworker = len(env.roledefs['django_app']) > 1

    if more_than_one_webworker:
        execute(_decommission_host, host)

    yield

    if more_than_one_webworker:
        execute(_recommission_host, host) 

def get_roles_for(host):
    roles = []
    for role in ['coordinator', 'worker']:
        if host in env.roledefs[role]:
            roles.append(role)
    return roles 

def set_env_from_conf(self, conf):
        self.config = conf
        env.user = conf['username']
        env.port = conf['port']
        try:
            env.java8_home = conf['java8_home']
        except KeyError:
            env.java8_home = None
        env.roledefs['coordinator'] = [conf['coordinator']]
        env.roledefs['worker'] = conf['workers']
        env.roledefs['all'] = self._dedup_list(util.get_coordinator_role() +
                                               util.get_worker_role())

        env.hosts = env.roledefs['all'][:]
        env.conf = conf 

def default_config(self, filename):
        try:
            conf = copy.deepcopy(self.DEFAULT_PROPERTIES[filename])
        except KeyError:
            raise ConfigurationError('Invalid configuration file name: %s' %
                                     filename)
        if filename == 'config.properties':
            coordinator = env.roledefs['coordinator'][0]
            workers = env.roledefs['worker']
            if coordinator in workers:
                conf['node-scheduler.include-coordinator'] = 'true'
            conf['discovery.uri'] = 'http://%s:8080' % coordinator
        return conf 

def get_coordinator_role():
    return env.roledefs['coordinator'] 

def by_rolename(host, rolename, f, *args, **kwargs):
    if rolename is None:
        f(*args, **kwargs)
    else:
        if rolename not in env.roledefs.keys():
            abort("Invalid role name %s. Valid rolenames are %s" %
                  (rolename, env.roledefs.keys()))
        if host in env.roledefs[rolename]:
            return f(*args, **kwargs) 

def test_coordinator(self, coord_mock, configure_mock):
        env.roledefs['coordinator'] = ['master']
        env.host = 'master'
        deploy.coordinator()
        assert configure_mock.called 

def test_check_success_fail(self, mock_query_for_status, string_config_mock, mock_run,
                                mock_presto_config):
        env.roledefs = {
            'coordinator': ['Node1'],
            'worker': ['Node1', 'Node2', 'Node3', 'Node4'],
            'all': ['Node1', 'Node2', 'Node3', 'Node4']
        }
        env.hosts = env.roledefs['all']
        env.host = 'Node1'
        string_config_mock.return_value = 'Node1'
        mock_query_for_status.return_value = False
        self.assertEqual(server.check_server_status(), False)