Python oslo_utils.encodeutils.safe_decode() Examples

def get_query_param(req, param_name, required=False, default_val=None):
    try:
        params = falcon.uri.parse_query_string(req.query_string)
        if param_name in params:
            if isinstance(params[param_name], list):
                param_val = encodeutils.safe_decode(params[param_name][0], 'utf8')
            else:
                param_val = encodeutils.safe_decode(params[param_name], 'utf8')

            return param_val
        else:
            if required:
                raise Exception("Missing " + param_name)
            else:
                return default_val
    except Exception as ex:
        LOG.debug(ex)
        raise HTTPUnprocessableEntityError('Unprocessable Entity', str(ex)) 

def decrypt(data, encryption_key=None):
    if data is None:
        return None

    encryption_key = get_valid_encryption_key(encryption_key)
    encoded_key = base64.b64encode(encryption_key.encode('utf-8'))
    sym = fernet.Fernet(encoded_key)
    try:
        value = sym.decrypt(encodeutils.safe_encode(data))
        if value is not None:
            return encodeutils.safe_decode(value, 'utf-8')
    except fernet.InvalidToken:
        raise exception.InvalidEncryptionKey() 

def test_call(self, mock_v1):
        mock_v1.return_value = {'foo': 'bar'}
        conf = mock.Mock()
        controller = versions.Controller(conf)
        environ = {
            'REQUEST_METHOD': 'GET',
            'SERVER_NAME': 'host',
            'SERVER_PORT': 8778,
            'SCRIPT_NAME': '/',
            'PATH_INFO': '/',
            'wsgi.url_scheme': 'http',
        }
        req = wsgi.Request(environ)
        expected_dict = {
            'versions': [{'foo': 'bar'}]
        }
        expected_body = jsonutils.dumps(expected_dict)

        resp = controller(req)

        self.assertIsInstance(resp, webob.Response)
        self.assertEqual(expected_body, encodeutils.safe_decode(resp.body))
        self.assertEqual(http_client.MULTIPLE_CHOICES, resp.status_code)
        self.assertEqual('application/json', resp.content_type) 

def _run_proc(cmd):
    """Simple wrapper to run a process.

    Will return the return code along with the stdout and stderr.  It is the
    decision of the caller if it wishes to honor or ignore the return code.

    :return: The return code, stdout and stderr from the command.
    """
    process = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               close_fds=True, env=None)
    process.wait()
    stdout, stderr = process.communicate()
    # Convert the stdout/stderr output from a byte-string to a unicode-string
    # so it doesn't blow up later on anything doing an implicit conversion
    stdout = encodeutils.safe_decode(stdout)
    stderr = encodeutils.safe_decode(stderr)
    return process.returncode, stdout, stderr 

def test_safe_decode(self):
        safe_decode = encodeutils.safe_decode
        self.assertRaises(TypeError, safe_decode, True)
        self.assertEqual(six.u('ni\xf1o'), safe_decode(six.b("ni\xc3\xb1o"),
                         incoming="utf-8"))
        if six.PY2:
            # In Python 3, bytes.decode() doesn't support anymore
            # bytes => bytes encodings like base64
            self.assertEqual(six.u("test"), safe_decode("dGVzdA==",
                             incoming='base64'))

        self.assertEqual(six.u("strange"), safe_decode(six.b('\x80strange'),
                         errors='ignore'))

        self.assertEqual(six.u('\xc0'), safe_decode(six.b('\xc0'),
                         incoming='iso-8859-1'))

        # Forcing incoming to ascii so it falls back to utf-8
        self.assertEqual(six.u('ni\xf1o'), safe_decode(six.b('ni\xc3\xb1o'),
                         incoming='ascii'))

        self.assertEqual(six.u('foo'), safe_decode(b'foo')) 

def test_resource_call_with_version_header(self):
        class Controller(object):
            def dance(self, req):
                return {'foo': 'bar'}

        actions = {'action': 'dance'}
        env = {'wsgiorg.routing_args': [None, actions]}
        request = wsgi.Request.blank('/tests/123', environ=env)
        request.version_request = vr.APIVersionRequest('1.0')

        resource = wsgi.Resource(Controller())
        resp = resource(request)
        self.assertEqual('{"foo": "bar"}', encodeutils.safe_decode(resp.body))
        self.assertTrue(hasattr(resp, 'headers'))
        expected = 'clustering 1.0'
        self.assertEqual(expected, resp.headers['OpenStack-API-Version'])
        self.assertEqual('OpenStack-API-Version', resp.headers['Vary']) 

def test_req_version_matches_with_None_None(self):
        version_request = api_version_request.APIVersionRequest('2.39')
        self.mock_object(api_version_request,
                         'max_api_version',
                         mock.Mock(return_value=version_request))

        class Controller(wsgi.Controller):

            def index(self, req):
                req_version = req.api_version_request
                # This case is artificial, and will return True
                if req_version.matches(None, None):
                    return "Pass"

        req = fakes.HTTPRequest.blank('/tests', base_url='http://localhost/v2')
        req.headers = {version_header_name: '2.39'}
        app = fakes.TestRouter(Controller())

        response = req.get_response(app)

        resp = encodeutils.safe_decode(response.body, incoming='utf-8')
        self.assertEqual("Pass", resp)
        self.assertEqual(200, response.status_int) 

def test_req_version_matches_with_None(self, version, ret_val):
        version_request = api_version_request.APIVersionRequest(version)
        self.mock_object(api_version_request,
                         'max_api_version',
                         mock.Mock(return_value=version_request))

        class Controller(wsgi.Controller):

            def index(self, req):
                req_version = req.api_version_request
                if req_version.matches(None, '2.8'):
                    return 'older'
                if req_version.matches('2.9', None):
                    return 'newer'

        req = fakes.HTTPRequest.blank('/tests', base_url='http://localhost/v2')
        req.headers = {version_header_name: version}
        app = fakes.TestRouter(Controller())

        response = req.get_response(app)

        resp = encodeutils.safe_decode(response.body, incoming='utf-8')
        self.assertEqual(ret_val, resp)
        self.assertEqual(200, response.status_int) 

def test_req_version_matches_with_if(self, version, ret_val):
        version_request = api_version_request.APIVersionRequest(version)
        self.mock_object(api_version_request,
                         'max_api_version',
                         mock.Mock(return_value=version_request))

        class Controller(wsgi.Controller):

            def index(self, req):
                req_version = req.api_version_request
                if req_version.matches('2.1', '2.8'):
                    return 'older'
                if req_version.matches('2.9', '2.88'):
                    return 'newer'

        req = fakes.HTTPRequest.blank('/tests', base_url='http://localhost/v2')
        req.headers = {version_header_name: version}
        app = fakes.TestRouter(Controller())

        response = req.get_response(app)

        resp = encodeutils.safe_decode(response.body, incoming='utf-8')
        self.assertEqual(ret_val, resp)
        self.assertEqual(200, response.status_int) 

def get_workers(function_id, version=0):
    client = get_client()
    values = client.get_prefix("%s_%s/worker" % (function_id, version))
    workers = [encodeutils.safe_decode(w[0]) for w in values]
    return workers 

def paginate(resource, uri, limit):
    parsed_uri = urlparse.urlparse(uri)

    self_link = encodeutils.safe_decode(build_base_uri(parsed_uri), 'utf8')

    old_query_params = _get_old_query_params(parsed_uri)

    if old_query_params:
        self_link += '?' + '&'.join(old_query_params)

    if resource and len(resource) > limit:

        if 'id' in resource[limit - 1]:
            new_offset = resource[limit - 1]['id']

        next_link = encodeutils.safe_decode(build_base_uri(parsed_uri), 'utf8')

        new_query_params = [u'offset' + '=' + urlparse.quote(
            new_offset.encode('utf8'), safe='')]

        _get_old_query_params_except_offset(new_query_params, parsed_uri)

        if new_query_params:
            next_link += '?' + '&'.join(new_query_params)

        resource = {u'links': ([{u'rel': u'self',
                                 u'href': self_link},
                                {u'rel': u'next',
                                 u'href': next_link}]),
                    u'elements': resource[:limit]}

    else:

        resource = {u'links': ([{u'rel': u'self',
                                 u'href': encodeutils.safe_decode(self_link, 'utf-8')}]),
                    u'elements': resource}

    return resource 

def paginate_alarming(resource, uri, limit):
    parsed_uri = urlparse.urlparse(uri)

    self_link = build_base_uri(parsed_uri)

    old_query_params = _get_old_query_params(parsed_uri)

    if old_query_params:
        self_link += '?' + '&'.join(old_query_params)

    if resource and len(resource) > limit:

        old_offset = 0
        for param in old_query_params:
            if param.find('offset') >= 0:
                old_offset = int(param.split('=')[-1])
        new_offset = str(limit + old_offset)

        next_link = build_base_uri(parsed_uri)

        new_query_params = [u'offset' + '=' + urlparse.quote(
            new_offset.encode('utf8'), safe='')]

        _get_old_query_params_except_offset(new_query_params, parsed_uri)

        if new_query_params:
            next_link += '?' + '&'.join(new_query_params)

        resource = {u'links': ([{u'rel': u'self',
                                 u'href': encodeutils.safe_decode(self_link, 'utf8')},
                                {u'rel': u'next',
                                 u'href': encodeutils.safe_decode(next_link, 'utf8')}]),
                    u'elements': resource[:limit]}

    else:

        resource = {u'links': ([{u'rel': u'self',
                                 u'href': encodeutils.safe_decode(self_link, 'utf8')}]),
                    u'elements': resource}

    return resource 

def on_get(self, req, res):
        helpers.validate_authorization(req,
                                       ['api:versions'])
        result = {
            'id': 'v2.0',
            'links': [{
                'rel': 'self',
                'href': encodeutils.safe_decode(req.uri, 'utf-8')
            }],
            'status': 'CURRENT',
            'updated': "2013-03-06T00:00:00.000Z"
        }
        res.body = helpers.to_json(result) 

def main(argv=None):
    """If necessary, create the database, retention policy, and users"""
    auth_str = '?u=%s&p=%s' % (ADMIN, ADMIN_PASS)
    api_uri = "{0}/query{1}".format(URL, auth_str)

    # List current databases
    dbs = influxdb_get(uri=api_uri, query="SHOW DATABASES")
    if [DBNAME] not in dbs:
        print("Creating database '{}'".format(DBNAME))
        influxdb_get_post(uri=api_uri, query="CREATE DATABASE {0}".format(DBNAME))
        print("...created!")

    # Check retention policy
    policies = influxdb_get(uri=api_uri,
                            query="SHOW RETENTION POLICIES ON {0}".format(DBNAME))
    if not any(pol[0] == SHARDSPACE_NAME for pol in policies):
        # Set retention policy
        policy = ("CREATE RETENTION POLICY {0} ON {1} DURATION {2} "
                  "REPLICATION {3} DEFAULT".format(SHARDSPACE_NAME,
                                                   DBNAME,
                                                   RETENTION,
                                                   REPLICATION)
                  )
        influxdb_get_post(uri=api_uri, db=DBNAME, query=policy)

    # Create the users
    users = influxdb_get(uri=api_uri, query="SHOW USERS", db=DBNAME)
    for name, password in USERS.items():
        if not any(user[0] == name for user in users):
            influxdb_get_post(uri=api_uri,
                              query=safe_decode("CREATE USER {0} WITH PASSWORD '{1}'"
                                                .format(name, password)),
                              db=DBNAME) 

def test_default(self):
        fixture = {"key": "value"}
        response = webob.Response()
        serializers.JSONResponseSerializer().default(response, fixture)
        self.assertEqual(200, response.status_int)
        content_types = [h for h in response.headerlist
                         if h[0] == 'Content-Type']
        # NOTE: filter returns a iterator in python 3.
        types = [t for t in content_types]
        self.assertEqual(1, len(types))
        self.assertEqual('application/json', response.content_type)
        self.assertEqual('{"key": "value"}',
                         encodeutils.safe_decode(response.body)) 

def get_comma_separated_str_as_list(comma_separated_str):
    if not comma_separated_str:
        return []
    else:
        return encodeutils.safe_decode(comma_separated_str, 'utf-8').split(',') 

def get_service_url(function_id, version=0):
    client = get_client()
    values = client.get('%s_%s/service_url' % (function_id, version))
    return None if not values else encodeutils.safe_decode(values[0]) 

def to_slug(value, incoming=None, errors="strict"):
    """Normalize string.

    Convert to lowercase, remove non-word characters, and convert spaces
    to hyphens.

    Inspired by Django's `slugify` filter.

    :param value: Text to slugify
    :param incoming: Text's current encoding
    :param errors: Errors handling policy. See here for valid
        values http://docs.python.org/2/library/codecs.html
    :returns: slugified unicode representation of `value`
    :raises TypeError: If text is not an instance of str
    """
    value = encodeutils.safe_decode(value, incoming, errors)
    # NOTE(aababilov): no need to use safe_(encode|decode) here:
    # encodings are always "ascii", error handling is always "ignore"
    # and types are always known (first: unicode; second: str)
    value = unicodedata.normalize("NFKD", value).encode(
        "ascii", "ignore").decode("ascii")
    value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
    return SLUGIFY_HYPHENATE_RE.sub("-", value)


# NOTE(dhellmann): Before submitting a patch to add a new argument to
# this function to allow the caller to pass in "extra" or "additional"
# or "replacement" patterns to be masked out, please note that we have
# discussed that feature many times and always rejected it based on
# the desire to have Oslo functions behave consistently across all
# projects and *especially* to have security features work the same
# way no matter where they are used. If every project adopted its own
# set patterns for secret values, it would be very difficult to audit
# the logging to ensure that everything is properly masked. So, please
# either add your pattern to the module-level variables at the top of
# this file or, even better, pick an existing pattern or key to use in
# your application to ensure that the value is masked by this
# function. 

def test_safe_encode_same_encoding_different_cases(self):
        with mock.patch.object(encodeutils, 'safe_decode', mock.Mock()):
            utf8 = encodeutils.safe_encode(
                six.u('foo\xf1bar'), encoding='utf-8')
            self.assertEqual(
                encodeutils.safe_encode(utf8, 'UTF-8', 'utf-8'),
                encodeutils.safe_encode(utf8, 'utf-8', 'UTF-8'),
            )
            self.assertEqual(
                encodeutils.safe_encode(utf8, 'UTF-8', 'utf-8'),
                encodeutils.safe_encode(utf8, 'utf-8', 'utf-8'),
            )
            encodeutils.safe_decode.assert_has_calls([]) 

def main():
    try:
        OpenStackMagnumShell().main(map(encodeutils.safe_decode, sys.argv[1:]))

    except Exception as e:
        logger.debug(e, exc_info=1)
        print("ERROR: %s" % encodeutils.safe_encode(str(e)),
              file=sys.stderr)
        sys.exit(1) 

def main():
    try:
        argv = [encodeutils.safe_decode(a) for a in sys.argv[1:]]
        HarborShell().main(argv)
    except KeyboardInterrupt:
        print("... terminating harbor client", file=sys.stderr)
        sys.exit(130)
    except exc.CommandError as e:
        print("CommandError: %s" % e)
        sys.exit(127) 

def args2body(self, parsed_args):
        body = {_VNF: {}}
        body[_VNF]['attributes'] = {}

        config = None
        if parsed_args.config_file:
            with open(parsed_args.config_file) as f:
                config_yaml = f.read()
            try:
                config = yaml.load(config_yaml, Loader=yaml.SafeLoader)
            except yaml.YAMLError as e:
                raise exceptions.InvalidInput(reason=e)
            if not config:
                raise exceptions.InvalidInput(
                    reason='The config file is empty')
        if parsed_args.config:
            decoded_config = encodeutils.safe_decode(parsed_args.config)
            try:
                config = yaml.load(decoded_config, Loader=yaml.SafeLoader)
            except yaml.YAMLError as e:
                raise exceptions.InvalidInput(reason=e)
            if not config:
                raise exceptions.InvalidInput(
                    reason='The parameter is empty')
        if config:
            body[_VNF]['attributes'] = {'config': config}
        if parsed_args.param_file:
            with open(parsed_args.param_file) as f:
                param_yaml = f.read()
            try:
                param = yaml.load(
                    param_yaml, Loader=yaml.SafeLoader)
            except yaml.YAMLError as e:
                raise exceptions.InvalidInput(reason=e)
            if not param:
                raise exceptions.InvalidInput(
                    reason='The parameter file is empty')
            body[_VNF]['attributes'] = {'param_values': param}
        tackerV10.update_dict(parsed_args, body[_VNF], ['tenant_id'])
        return body 

def exit(msg=None, exit_code=1):
    if msg:
        print(encodeutils.safe_decode(msg))
    sys.exit(exit_code) 

def args2body(self, parsed_args):
        body = {self.resource: {}}
        # config arg passed as data overrides config yaml when both args passed
        config = None
        if parsed_args.config_file:
            with open(parsed_args.config_file) as f:
                config_yaml = f.read()
            try:
                config = yaml.load(config_yaml, Loader=yaml.SafeLoader)
            except yaml.YAMLError as e:
                raise exceptions.InvalidInput(reason=e)
            if not config:
                raise exceptions.InvalidInput(
                    reason='The config file is empty')
        if parsed_args.config:
            config_param = encodeutils.safe_decode(parsed_args.config)
            try:
                config = yaml.load(config_param, Loader=yaml.SafeLoader)
            except yaml.YAMLError as e:
                raise exceptions.InvalidInput(reason=e)
            if not config:
                raise exceptions.InvalidInput(
                    reason='The parameter is empty')
        if config:
            body[self.resource]['attributes'] = {'config': config}
        if parsed_args.param_file:
            with open(parsed_args.param_file) as f:
                param_yaml = f.read()
            try:
                param = yaml.load(
                    param_yaml, Loader=yaml.SafeLoader)
            except yaml.YAMLError as e:
                raise exceptions.InvalidInput(reason=e)
            if not param:
                raise exceptions.InvalidInput(
                    reason='The parameter file is empty')
            body[self.resource]['attributes'] = {'param_values': param}
        tackerV10.update_dict(parsed_args, body[self.resource], ['tenant_id'])
        return body 

def main(argv=sys.argv[1:]):
    try:
        return TackerShell(TACKER_API_VERSION).run(
            list(map(encodeutils.safe_decode, argv)))
    except KeyboardInterrupt:
        print("... terminating tacker client", file=sys.stderr)
        return 130
    except exc.TackerClientException:
        return 1
    except Exception as e:
        print(e)
        return 1 

def main():
    CONF.register_cli_opt(command_opt)
    if len(sys.argv) < 2:
        script_name = sys.argv[0]
        print("%s command action [<args>]" % script_name)
        print(_("Available commands:"))
        for command in COMMANDS:
            print(_("\t%s") % command)
        sys.exit(2)

    try:
        logging.register_options(CONF)

        cfg_files = cfg.find_config_files(project='searchlight',
                                          prog='searchlight')
        config.parse_args(default_config_files=cfg_files)
        config.set_config_defaults()
        logging.setup(CONF, 'searchlight')

        func_kwargs = {}
        for k in CONF.command.action_kwargs:
            v = getattr(CONF.command, 'action_kwarg_' + k)
            if v is None:
                continue
            if isinstance(v, str):
                v = encodeutils.safe_decode(v)
            func_kwargs[k] = v
        func_args = [encodeutils.safe_decode(arg)
                     for arg in CONF.command.action_args]
        return CONF.command.action_fn(*func_args, **func_kwargs)

    except RuntimeError as e:
        sys.exit("ERROR: %s" % e) 

def encrypt(value, encryption_key=None):
    if value is None:
        return None

    encryption_key = get_valid_encryption_key(encryption_key)
    encoded_key = base64.b64encode(encryption_key.encode('utf-8'))
    sym = fernet.Fernet(encoded_key)
    res = sym.encrypt(encodeutils.safe_encode(value))
    return encodeutils.safe_decode(res) 

def process_result_value(self, value, dialect):
        return encodeutils.safe_decode(zlib.decompress(value)) 

def base64_encode_psk(self):
        if not self.vpnservice:
            return
        for ipsec_site_conn in self.vpnservice['ipsec_site_connections']:
            psk = ipsec_site_conn['psk']
            encoded_psk = base64.b64encode(encodeutils.safe_encode(psk))
            # NOTE(huntxu): base64.b64encode returns an instance of 'bytes'
            # in Python 3, convert it to a str. For Python 2, after calling
            # safe_decode, psk is converted into a unicode not containing any
            # non-ASCII characters so it doesn't matter.
            psk = encodeutils.safe_decode(encoded_psk, incoming='utf_8')
            ipsec_site_conn['psk'] = PSK_BASE64_PREFIX + psk 

def _safe_header(self, name, value):
        if name in SENSITIVE_HEADERS:
            # because in python3 byte string handling is ... ug
            v = value.encode('utf-8')
            h = hashlib.sha1(v)
            d = h.hexdigest()
            return encodeutils.safe_decode(name), "{SHA1}%s" % d
        else:
            return (encodeutils.safe_decode(name),
                    encodeutils.safe_decode(value))