Python ast.JoinedStr() Examples

def _evaluate_ast(node):
    wrapper = None
    statement = ''

    if isinstance(node._bandit_parent, ast.BinOp):
        out = utils.concat_string(node, node._bandit_parent)
        wrapper = out[0]._bandit_parent
        statement = out[1]
    elif (isinstance(node._bandit_parent, ast.Attribute)
          and node._bandit_parent.attr == 'format'):
        statement = node.s
        # Hierarchy for "".format() is Wrapper -> Call -> Attribute -> Str
        wrapper = node._bandit_parent._bandit_parent._bandit_parent
    elif (hasattr(ast, 'JoinedStr')
          and isinstance(node._bandit_parent, ast.JoinedStr)):
        statement = node.s
        wrapper = node._bandit_parent._bandit_parent

    if isinstance(wrapper, ast.Call):  # wrapped in "execute" call?
        names = ['execute', 'executemany']
        name = utils.get_called_name(wrapper)
        return (name in names, statement)
    else:
        return (False, statement) 

def visit_Call(self, node: ast.Call):
        """
        Convert `ast.Call` to `ast.JoinedStr` f-string
        """

        match = matching_call(node)

        if match:
            state.call_candidates += 1

            # bail in these edge cases...
            if any(isinstance(arg, ast.Starred) for arg in node.args):
                return node

            result_node = joined_string(node)
            self.visit(result_node)
            self.counter += 1
            state.call_transforms += 1
            return result_node

        return node 

def supports_feature(feature):
  if feature == 'bytes_node':
    return hasattr(ast, 'Bytes') and issubclass(ast.Bytes, ast.AST)
  if feature == 'exec_node':
    return hasattr(ast, 'Exec') and issubclass(ast.Exec, ast.AST)
  if feature == 'type_annotations':
    try:
      ast.parse('def foo(bar: str=123) -> None: pass')
    except SyntaxError:
      return False
    return True
  if feature == 'fstring':
    return hasattr(ast, 'JoinedStr') and issubclass(ast.JoinedStr, ast.AST)
  # Python 2 counts tabs as 8 spaces for indentation
  if feature == 'mixed_tabs_spaces':
    return sys.version_info[0] < 3
  return False 

def _transform_string(self, node: ast.JoinedStr) -> ast.Call:
        htm_strings: List[str] = []
        exp_nodes: List[ast.AST] = []
        for inner_node in node.values:
            if isinstance(inner_node, ast.Str):
                htm_strings.append(inner_node.s)
            elif isinstance(inner_node, ast.FormattedValue):
                if len(htm_strings) == len(exp_nodes):
                    htm_strings.append("")
                if inner_node.conversion != -1 or inner_node.format_spec:
                    exp_nodes.append(ast.JoinedStr([inner_node]))
                else:
                    exp_nodes.append(inner_node.value)

        call_stack = _HtmlCallStack()
        for op_type, *data in htm.htm_parse(htm_strings):
            getattr(self, f"_transform_htm_{op_type.lower()}")(
                exp_nodes, call_stack, *data
            )
        return call_stack.finish() 

def visit_Call(self, node: ast.Call) -> Optional[ast.AST]:
        if isinstance(node.func, ast.Name):
            if node.func.id == "html":
                if (
                    not node.keywords
                    and len(node.args) == 1
                    and isinstance(node.args[0], ast.JoinedStr)
                ):
                    try:
                        new_node = self._transform_string(node.args[0])
                    except htm.ParseError as error:
                        raise DialectError(str(error), self.filename, node.lineno)
                    return self.generic_visit(
                        ast.fix_missing_locations(ast.copy_location(new_node, node))
                    )
        return node 

def transform_generic(node):
    """Convert a `BinOp` `%` formatted str with a unknown name on the `node.right` to an f-string.

    When `node.right` is a Name since we don't know if it's a single var or a dict so we sniff the string.

    Sniffs the left string for Dict style usage
    e.g. `"val: %(key_name1)s val2: %(key_name2)s" % some_dict`

    else (e.g. `"val: %s" % some_var`):
    Borrow the core logic by injecting the name into a ast.Tuple

    Returns ast.JoinedStr (f-string), bool: str-in-str
    """

    has_dict_str_format = DICT_PATTERN.findall(node.left.s)
    if has_dict_str_format:
        return transform_dict(node), True

    # if it's just a name then pretend it's tuple to use that code
    node.right = ast.Tuple(elts=[node.right])
    return transform_tuple(node), False 

def ast_formatted_value(
    val, fmt_str: str = None, conversion=None
) -> ast.FormattedValue:
    if isinstance(val, ast.FormattedValue):
        return val

    if astor.to_source(val)[0] == "{":
        raise FlyntException(
            "values starting with '{' are better left not transformed."
        )

    if fmt_str:
        format_spec = ast.JoinedStr([ast_string_node(fmt_str.replace(":", ""))])
    else:
        format_spec = None

    if conversion is None:
        conversion = -1
    else:
        conversion = ord(conversion.replace("!", ""))
    return ast.FormattedValue(value=val, conversion=conversion, format_spec=format_spec) 

def visit_JoinedStr(self, node):
        new_vals = []
        for v in node.values:
            if (
                isinstance(v, ast.FormattedValue)
                and isinstance(v.value, ast.JoinedStr)
                and v.format_spec is None
            ):
                new_vals += v.value.values
            else:
                new_vals.append(v)

        node.values = new_vals
        return self.generic_visit(node) 

def visit_JoinedStr(self, node: ast.JoinedStr) -> None:
        """
        Forbids to use ``f`` strings.

        Raises:
            FormattedStringViolation

        """
        self.add_violation(consistency.FormattedStringViolation(node))
        self.generic_visit(node) 

def joined_str(d: ast.JoinedStr):
    return "+".join(find(value) for value in d.values) 

def is_str_literal(node):
    """ Returns True if a node is a string literal """
    if isinstance(node, ast.Constant):
        return isinstance(node.value, str)
    elif isinstance(node, ast.JoinedStr):
        return True
    else:
        return False 

def visit_BinOp(self, node: ast.BinOp):
        """
        Transforms a string concat to an f-string
        """
        if is_string_concat(node):
            self.counter += 1
            left, right = node.left, node.right
            left = self.visit(left)
            right = self.visit(right)

            if not check_sns_depth(left) or not check_sns_depth(right):
                node.left = left
                node.right = right
                return node

            parts = []
            for p in [left, right]:
                if isinstance(p, ast.JoinedStr):
                    parts += p.values
                else:
                    parts.append(p)

            segments = []
            for p in parts:
                if isinstance(p, ast.Constant):
                    segments.append(ast_string_node(p.value))
                else:
                    segments.append(ast_formatted_value(p))

            return ast.JoinedStr(segments)
        else:
            return self.generic_visit(node) 

def test_ast_line_numbers(self):
        expr = """
a = 10
f'{a * x()}'"""
        t = ast.parse(expr)
        self.assertEqual(type(t), ast.Module)
        self.assertEqual(len(t.body), 2)
        # check `a = 10`
        self.assertEqual(type(t.body[0]), ast.Assign)
        self.assertEqual(t.body[0].lineno, 2)
        # check `f'...'`
        self.assertEqual(type(t.body[1]), ast.Expr)
        self.assertEqual(type(t.body[1].value), ast.JoinedStr)
        self.assertEqual(len(t.body[1].value.values), 1)
        self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue)
        self.assertEqual(t.body[1].lineno, 3)
        self.assertEqual(t.body[1].value.lineno, 3)
        self.assertEqual(t.body[1].value.values[0].lineno, 3)
        # check the binop location
        binop = t.body[1].value.values[0].value
        self.assertEqual(type(binop), ast.BinOp)
        self.assertEqual(type(binop.left), ast.Name)
        self.assertEqual(type(binop.op), ast.Mult)
        self.assertEqual(type(binop.right), ast.Call)
        self.assertEqual(binop.lineno, 3)
        self.assertEqual(binop.left.lineno, 3)
        self.assertEqual(binop.right.lineno, 3)
        self.assertEqual(binop.col_offset, 3)
        self.assertEqual(binop.left.col_offset, 3)
        self.assertEqual(binop.right.col_offset, 7) 

def transform_tuple(node):
    """Convert a `BinOp` `%` formatted str with a tuple on the right to an f-string.

    Takes an ast.BinOp representing `"1. %s 2. %s" % (a, b)`
    and converted it to a ast.JoinedStr representing `f"1. {a} 2. {b}"`

    Args:
       node (ast.BinOp): The node to convert to a f-string

    Returns ast.JoinedStr (f-string)
    """

    format_str = node.left.s
    matches = VAR_KEY_PATTERN.findall(format_str)

    if len(node.right.elts) != len(matches):
        raise FlyntException("string formatting length mismatch")

    str_vars = deque(node.right.elts)

    segments = []
    blocks = deque(VAR_KEY_PATTERN.split(format_str))
    segments.append(ast_string_node(blocks.popleft().replace("%%", "%")))

    while len(blocks) > 0:

        fmt_prefix = blocks.popleft()
        fmt_spec = blocks.popleft()
        val = str_vars.popleft()

        fv = formatted_value(fmt_prefix, fmt_spec, val)

        segments.append(fv)
        segments.append(ast_string_node(blocks.popleft().replace("%%", "%")))

    return ast.JoinedStr(segments) 

def str_maker(*strs: Tokenizer):
    head = strs[0]

    return ast.JoinedStr(**(loc @ head), values=list(map(_parse_expr, strs))) 

def test_ast_line_numbers_multiline_fstring(self):
        # FIXME: This test demonstrates invalid behavior due to JoinedStr's
        # immediate child nodes containing the wrong lineno.  The enclosed
        # expressions have valid line information and column offsets.
        # See bpo-16806 and bpo-30465 for details.
        expr = """
a = 10
f'''
  {a
     *
       x()}
non-important content
'''
"""
        t = ast.parse(expr)
        self.assertEqual(type(t), ast.Module)
        self.assertEqual(len(t.body), 2)
        # check `a = 10`
        self.assertEqual(type(t.body[0]), ast.Assign)
        self.assertEqual(t.body[0].lineno, 2)
        # check `f'...'`
        self.assertEqual(type(t.body[1]), ast.Expr)
        self.assertEqual(type(t.body[1].value), ast.JoinedStr)
        self.assertEqual(len(t.body[1].value.values), 3)
        self.assertEqual(type(t.body[1].value.values[0]), ast.Str)
        self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue)
        self.assertEqual(type(t.body[1].value.values[2]), ast.Str)
        # NOTE: the following invalid behavior is described in bpo-16806.
        # - line number should be the *first* line (3), not the *last* (8)
        # - column offset should not be -1
        self.assertEqual(t.body[1].lineno, 8)
        self.assertEqual(t.body[1].value.lineno, 8)
        self.assertEqual(t.body[1].value.values[0].lineno, 8)
        self.assertEqual(t.body[1].value.values[1].lineno, 8)
        self.assertEqual(t.body[1].value.values[2].lineno, 8)
        self.assertEqual(t.body[1].col_offset, -1)
        self.assertEqual(t.body[1].value.col_offset, -1)
        self.assertEqual(t.body[1].value.values[0].col_offset, -1)
        self.assertEqual(t.body[1].value.values[1].col_offset, -1)
        self.assertEqual(t.body[1].value.values[2].col_offset, -1)
        # NOTE: the following lineno information and col_offset is correct for
        # expressions within FormattedValues.
        binop = t.body[1].value.values[1].value
        self.assertEqual(type(binop), ast.BinOp)
        self.assertEqual(type(binop.left), ast.Name)
        self.assertEqual(type(binop.op), ast.Mult)
        self.assertEqual(type(binop.right), ast.Call)
        self.assertEqual(binop.lineno, 4)
        self.assertEqual(binop.left.lineno, 4)
        self.assertEqual(binop.right.lineno, 6)
        self.assertEqual(binop.col_offset, 3)
        self.assertEqual(binop.left.col_offset, 3)
        self.assertEqual(binop.right.col_offset, 7) 

def test_ast_line_numbers_multiple_formattedvalues(self):
        expr = """
f'no formatted values'
f'eggs {a * x()} spam {b + y()}'"""
        t = ast.parse(expr)
        self.assertEqual(type(t), ast.Module)
        self.assertEqual(len(t.body), 2)
        # check `f'no formatted value'`
        self.assertEqual(type(t.body[0]), ast.Expr)
        self.assertEqual(type(t.body[0].value), ast.JoinedStr)
        self.assertEqual(t.body[0].lineno, 2)
        # check `f'...'`
        self.assertEqual(type(t.body[1]), ast.Expr)
        self.assertEqual(type(t.body[1].value), ast.JoinedStr)
        self.assertEqual(len(t.body[1].value.values), 4)
        self.assertEqual(type(t.body[1].value.values[0]), ast.Str)
        self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue)
        self.assertEqual(type(t.body[1].value.values[2]), ast.Str)
        self.assertEqual(type(t.body[1].value.values[3]), ast.FormattedValue)
        self.assertEqual(t.body[1].lineno, 3)
        self.assertEqual(t.body[1].value.lineno, 3)
        self.assertEqual(t.body[1].value.values[0].lineno, 3)
        self.assertEqual(t.body[1].value.values[1].lineno, 3)
        self.assertEqual(t.body[1].value.values[2].lineno, 3)
        self.assertEqual(t.body[1].value.values[3].lineno, 3)
        # check the first binop location
        binop1 = t.body[1].value.values[1].value
        self.assertEqual(type(binop1), ast.BinOp)
        self.assertEqual(type(binop1.left), ast.Name)
        self.assertEqual(type(binop1.op), ast.Mult)
        self.assertEqual(type(binop1.right), ast.Call)
        self.assertEqual(binop1.lineno, 3)
        self.assertEqual(binop1.left.lineno, 3)
        self.assertEqual(binop1.right.lineno, 3)
        self.assertEqual(binop1.col_offset, 8)
        self.assertEqual(binop1.left.col_offset, 8)
        self.assertEqual(binop1.right.col_offset, 12)
        # check the second binop location
        binop2 = t.body[1].value.values[3].value
        self.assertEqual(type(binop2), ast.BinOp)
        self.assertEqual(type(binop2.left), ast.Name)
        self.assertEqual(type(binop2.op), ast.Add)
        self.assertEqual(type(binop2.right), ast.Call)
        self.assertEqual(binop2.lineno, 3)
        self.assertEqual(binop2.left.lineno, 3)
        self.assertEqual(binop2.right.lineno, 3)
        self.assertEqual(binop2.col_offset, 23)
        self.assertEqual(binop2.left.col_offset, 23)
        self.assertEqual(binop2.right.col_offset, 27) 

def test_ast_line_numbers_nested(self):
        expr = """
a = 10
f'{a * f"-{x()}-"}'"""
        t = ast.parse(expr)
        self.assertEqual(type(t), ast.Module)
        self.assertEqual(len(t.body), 2)
        # check `a = 10`
        self.assertEqual(type(t.body[0]), ast.Assign)
        self.assertEqual(t.body[0].lineno, 2)
        # check `f'...'`
        self.assertEqual(type(t.body[1]), ast.Expr)
        self.assertEqual(type(t.body[1].value), ast.JoinedStr)
        self.assertEqual(len(t.body[1].value.values), 1)
        self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue)
        self.assertEqual(t.body[1].lineno, 3)
        self.assertEqual(t.body[1].value.lineno, 3)
        self.assertEqual(t.body[1].value.values[0].lineno, 3)
        # check the binop location
        binop = t.body[1].value.values[0].value
        self.assertEqual(type(binop), ast.BinOp)
        self.assertEqual(type(binop.left), ast.Name)
        self.assertEqual(type(binop.op), ast.Mult)
        self.assertEqual(type(binop.right), ast.JoinedStr)
        self.assertEqual(binop.lineno, 3)
        self.assertEqual(binop.left.lineno, 3)
        self.assertEqual(binop.right.lineno, 3)
        self.assertEqual(binop.col_offset, 3)
        self.assertEqual(binop.left.col_offset, 3)
        self.assertEqual(binop.right.col_offset, 7)
        # check the nested call location
        self.assertEqual(len(binop.right.values), 3)
        self.assertEqual(type(binop.right.values[0]), ast.Str)
        self.assertEqual(type(binop.right.values[1]), ast.FormattedValue)
        self.assertEqual(type(binop.right.values[2]), ast.Str)
        self.assertEqual(binop.right.values[0].lineno, 3)
        self.assertEqual(binop.right.values[1].lineno, 3)
        self.assertEqual(binop.right.values[2].lineno, 3)
        call = binop.right.values[1].value
        self.assertEqual(type(call), ast.Call)
        self.assertEqual(call.lineno, 3)
        self.assertEqual(call.col_offset, 11) 

def joined_string(fmt_call: ast.Call) -> ast.JoinedStr:
    """ Transform a "...".format() call node into a f-string node. """
    string = fmt_call.func.value.s
    var_map = {kw.arg: kw.value for kw in fmt_call.keywords}

    for i, val in enumerate(fmt_call.args):
        var_map[i] = val

    splits = deque(stdlib_parse(string))

    seq_ctr = 0
    new_segments = []
    manual_field_ordering = False

    for raw, var_name, fmt_str, conversion in splits:
        if raw:
            new_segments.append(ast_string_node(raw))

        if var_name is None:
            continue

        if "[" in var_name:
            raise FlyntException(
                f"Skipping f-stringify of a fmt call with indexed name {var_name}"
            )

        suffix = ""
        if "." in var_name:
            idx = var_name.find(".")
            var_name, suffix = var_name[:idx], var_name[idx + 1 :]

        if var_name.isdigit():
            manual_field_ordering = True
            identifier = int(var_name)
        elif len(var_name) == 0:
            assert not manual_field_ordering
            identifier = seq_ctr
            seq_ctr += 1
        else:
            identifier = var_name

        try:
            ast_name = var_map.pop(identifier)
            if suffix:
                ast_name = ast.Attribute(value=ast_name, attr=suffix)
            new_segments.append(ast_formatted_value(ast_name, fmt_str, conversion))
        except KeyError as e:
            raise ConversionRefused(
                "A variable is used multiple times - better not to replace it."
            ) from e

    if var_map:
        raise FlyntException("A variable was never used - a risk of bug.")

    return ast.JoinedStr(new_segments) 

def transform_dict(node):
    """Convert a `BinOp` `%` formatted str with a name representing a Dict on the right to an f-string.

    Takes an ast.BinOp representing `"1. %(key1)s 2. %(key2)s" % mydict`
    and converted it to a ast.JoinedStr representing `f"1. {mydict['key1']} 2. {mydict['key2']}"`

    Args:
       node (ast.BinOp): The node to convert to a f-string

    Returns ast.JoinedStr (f-string)
    """

    format_str = node.left.s
    matches = DICT_PATTERN.findall(format_str)
    spec = []
    for idx, m in enumerate(matches):
        _, prefix, var_key, fmt_str, _ = SPLIT_DICT_PATTERN.split(m)
        if not var_key:
            raise FlyntException("could not find dict key")
        spec.append((prefix, var_key, fmt_str))

    # build result node
    segments = []
    spec.reverse()
    blocks = DICT_PATTERN.split(format_str)

    mapping = {}
    if isinstance(node.right, ast.Dict):
        for k, v in zip(node.right.keys, node.right.values):
            mapping[str(ast.literal_eval(k))] = v

        def make_fv(key: str):
            return mapping.pop(key)

    else:

        def make_fv(key: str):
            return ast.Subscript(
                value=node.right, slice=ast.Index(value=ast.Str(s=key))
            )

    for block in blocks:
        # if this block matches a %(arg)s pattern then inject f-string instead
        if DICT_PATTERN.match(block):
            prefix, var_key, fmt_str = spec.pop()
            fv = formatted_value(prefix, fmt_str, make_fv(var_key))
            segments.append(fv)
        else:
            # no match means it's just a literal string
            segments.append(ast.Str(s=block.replace("%%", "%")))

    if mapping:
        raise FlyntException("Not all keys were matched - probably an error.")

    return ast.JoinedStr(segments) 

def visit_BinOp(self, node):
        """Convert `ast.BinOp` to `ast.JoinedStr` f-string

        Currently only if a string literal `ast.Str` is on the left side of the `%`
        and one of `ast.Tuple`, `ast.Name`, `ast.Dict` is on the right

        Args:
            node (ast.BinOp): The node to convert to a f-string

        Returns ast.JoinedStr (f-string)
        """

        percent_stringify = (
            isinstance(node.left, ast.Str)
            and isinstance(node.op, ast.Mod)
            and isinstance(
                node.right,
                tuple( [ast.Tuple, ast.Dict] + supported_operands),
            )
        )

        if percent_stringify:
            state.percent_candidates += 1

            # bail in these edge cases...
            no_good = ["}", "{"]
            for ng in no_good:
                if ng in node.left.s:
                    return node
            for ch in ast.walk(node.right):
                # f-string expression part cannot include a backslash
                if isinstance(ch, ast.Str) and (
                    any(
                        map(
                            lambda x: x in ch.s,
                            ("\n", "\t", "\r", "'", '"', "%s", "%%"),
                        )
                    )
                    or "\\" in ch.s
                ):
                    return node

            result_node, str_in_str = transform_binop(node)
            self.string_in_string = str_in_str
            self.counter += 1
            state.percent_transforms += 1
            return result_node

        return node 

def get_kernel_embed():
        """A list of kernel embed nodes

        Returns:
            nodes (list): AST nodes which form the following code.

            ```
            import os
            pid = os.fork()
            if os.fork() == 0:
                open(f'{os.environ["HOME"]}/.pynt', 'a').close()
                import IPython
                IPython.start_kernel(user_ns={**locals(), **globals(), **vars()})
            os.waitpid(pid, 0)
            ```

        This is a purely functional method which always return the same thing.

        """
        return [
            ast.Import(names=[ast.alias(name='os', asname=None),]),
            ast.Assign(targets=[ast.Name(id='pid', ctx=ast.Store()),], value=ast.Call(func=ast.Attribute(value=ast.Name(id='os', ctx=ast.Load()), attr='fork', ctx=ast.Load()), args=[], keywords=[])),
            ast.If(
                test=ast.Compare(left=ast.Name(id='pid', ctx=ast.Load()), ops=[ast.Eq(),], comparators=[ast.Num(n=0),]),
                body=[
                    ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Call(func=ast.Name(id='open', ctx=ast.Load()), args=[
                        ast.JoinedStr(values=[
                            ast.FormattedValue(value=ast.Subscript(value=ast.Attribute(value=ast.Name(id='os', ctx=ast.Load()), attr='environ', ctx=ast.Load()), slice=ast.Index(value=ast.Str(s='HOME')), ctx=ast.Load()), conversion=-1, format_spec=None),
                            ast.Str(s='/.pynt'),
                        ]),
                        ast.Str(s='a'),
                    ], keywords=[]), attr='close', ctx=ast.Load()), args=[], keywords=[])),
                    ast.Import(names=[
                        ast.alias(name='IPython', asname=None),
                    ]),
                    ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id='IPython', ctx=ast.Load()), attr='start_kernel', ctx=ast.Load()), args=[], keywords=[
                        ast.keyword(arg='user_ns', value=ast.Dict(keys=[
                            None,
                            None,
                            None,
                        ], values=[
                            ast.Call(func=ast.Name(id='locals', ctx=ast.Load()), args=[], keywords=[]),
                            ast.Call(func=ast.Name(id='globals', ctx=ast.Load()), args=[], keywords=[]),
                            ast.Call(func=ast.Name(id='vars', ctx=ast.Load()), args=[], keywords=[]),
                        ])),
                    ])),
            ], orelse=[]),
            ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id='os', ctx=ast.Load()), attr='waitpid', ctx=ast.Load()), args=[
                ast.Name(id='pid', ctx=ast.Load()),
                ast.Num(n=0),
            ], keywords=[])),
        ]