Python tokenize.TokenInfo() Examples

def __init__(
        self,
        tree: ast.AST,
        file_tokens: Sequence[tokenize.TokenInfo],
        filename: str = constants.STDIN,
    ) -> None:
        """
        Creates new checker instance.

        These parameter names should not be changed.
        ``flake8`` has special API that passes concrete parameters to
        the plugins that ask for them.

        ``flake8`` also decides how to execute this plugin
        based on its parameters. This one is executed once per module.

        Arguments:
            tree: ``ast`` tree parsed by ``flake8``.
            file_tokens: ``tokenize.tokenize`` parsed file tokens.
            filename: module file name, might be empty if piping is used.

        """
        self.tree = transform(tree)
        self.filename = filename
        self.file_tokens = file_tokens 

def _check_executable_mismatch(
        self,
        token: tokenize.TokenInfo,
        *,
        is_shebang: bool,
    ) -> None:
        if is_windows() or self.filename == STDIN:
            # Windows does not have this concept of "executable" file.
            # The same for STDIN inputs.
            return

        is_executable = is_executable_file(self.filename)
        if is_executable and not is_shebang:
            self.add_violation(
                ShebangViolation(
                    text='file is executable but no shebang is present',
                ),
            )
        elif not is_executable and is_shebang:
            self.add_violation(
                ShebangViolation(
                    text='shebang is present but the file is not executable',
                ),
            ) 

def _check_valid_shebang(self, token: tokenize.TokenInfo) -> None:
        if self._python_executable not in token.line:
            self.add_violation(
                ShebangViolation(
                    text='shebang is present but does not contain `python`',
                ),
            )

        if token.start[1] != 0:
            self.add_violation(
                ShebangViolation(
                    text='there is a whitespace before shebang',
                ),
            )

        if token.start[0] != 1:
            self.add_violation(
                ShebangViolation(
                    text='there are blank or comment lines before shebang',
                ),
            ) 

def visit_string(self, token: tokenize.TokenInfo) -> None:
        """
        Finds incorrect string usages.

        ``u`` can only be the only prefix.
        You cannot combine it with ``r``, ``b``, or ``f``.
        Since it will raise a ``SyntaxError`` while parsing.

        Raises:
            UnicodeStringViolation
            WrongMultilineStringViolation
            ImplicitRawStringViolation
            WrongUnicodeEscapeViolation

        """
        self._check_correct_multiline(token)
        self._check_string_modifiers(token)
        self._check_implicit_raw_string(token)
        self._check_wrong_unicode_escape(token) 

def _check_string_modifiers(self, token: tokenize.TokenInfo) -> None:
        modifiers, _ = split_prefixes(token.string)

        if 'u' in modifiers.lower():
            self.add_violation(
                consistency.UnicodeStringViolation(token, text=token.string),
            )

        for mod in modifiers:
            if mod in self._bad_string_modifiers:
                self.add_violation(
                    consistency.UppercaseStringModifierViolation(
                        token,
                        text=mod,
                    ),
                ) 

def visit_number(self, token: tokenize.TokenInfo) -> None:
        """
        Checks number declarations.

        Raises:
            UnderscoredNumberViolation
            PartialFloatViolation
            BadNumberSuffixViolation
            BadComplexNumberSuffixViolation
            NumberWithMeaninglessZeroViolation
            PositiveExponentViolation
            FloatZeroViolation

        Regressions:
        https://github.com/wemake-services/wemake-python-styleguide/issues/557

        """
        self._check_complex_suffix(token)
        self._check_underscored_number(token)
        self._check_partial_float(token)
        self._check_bad_number_suffixes(token)
        self._check_float_zeros(token) 

def visit(self, token: tokenize.TokenInfo) -> None:
        """
        Runs custom defined handlers in a visitor for each specific token type.

        Uses ``.exact_type`` property to fetch the token name.
        So, you have to be extra careful with tokens
        like ``->`` and other operators,
        since they might resolve in just ``OP`` name.

        Does nothing if handler for any token type is not defined.

        Inspired by ``NodeVisitor`` class.

        See also:
            https://docs.python.org/3/library/tokenize.html

        """
        token_type = tokenize.tok_name[token.exact_type].lower()
        method = getattr(self, 'visit_{0}'.format(token_type), None)
        if method is not None:
            method(token) 

def fix_newlines(tokens):
    first = True
    curline = 1
    for token in tokens:
        if first:
            first = False
            curline = token.end[0] + 1
        else:
            # Fill NEWLINE token in between
            while curline < token.start[0]:
                yield TokenInfo(type=tokenize.NEWLINE,
                                string='\n',
                                start=(curline, 0),
                                end=(curline, 0),
                                line='\n', )
                curline += 1

            curline = token.end[0] + 1
        yield token 

def convert_toplevel_docstring(tokens):
    for token in tokens:
        # For each string
        if token.type == tokenize.STRING:
            text = token.string
            # Must be a docstring
            if text.startswith('"""') or text.startswith("'''"):
                startline, startcol = token.start
                # Starting column MUST be 0
                if startcol == 0:
                    endline, endcol = token.end
                    lines = ['# ' + line
                             for line in text.strip('"\' \n').split('\n')]
                    text = '\n'.join(lines)
                    fmt = '# <markdowncell>\n{0}\n# <codecell>'.format(text)
                    yield TokenInfo(type=tokenize.COMMENT,
                                    start=(startline, startcol),
                                    end=(endline, endcol),
                                    string=fmt,
                                    line='#')
                    # To next token
                    continue
        # Return untouched
        yield token 

def __init__(self, model, lstm_layers, lstm_size):
        self.__model = model

        # Initial state
        self._h0 = torch.zeros((lstm_layers, 1, lstm_size), device=device)
        self._c0 = torch.zeros((lstm_layers, 1, lstm_size), device=device)

        # Last line of source code read
        self._last_line = ""

        self._tokens: List[tokenize.TokenInfo] = []

        # Last token, because we need to input that to the model for inference
        self._last_token = 0

        # Last bit of the input string
        self._untokenized = ""

        # For timing
        self.time_add = 0
        self.time_predict = 0
        self.time_check = 0 

def __get_tokens(it):
        tokens: List[tokenize.TokenInfo] = []

        try:
            for t in it:
                if t.type in tokenizer.SKIP_TOKENS:
                    continue
                if t.type == tokenize.NEWLINE and t.string == '':
                    continue
                if t.type == tokenize.DEDENT:
                    continue
                if t.type == tokenize.ERRORTOKEN:
                    continue
                tokens.append(t)
        except tokenize.TokenError as e:
            if not e.args[0].startswith('EOF in'):
                print(e)
        except IndentationError as e:
            print(e)

        return tokens 

def _check_wrong_unicode_escape(self, token: tokenize.TokenInfo) -> None:
        # See: http://docs.python.org/reference/lexical_analysis.html
        modifiers, string_body = split_prefixes(token.string)

        index = 0
        while True:
            index = string_body.find('\\', index)
            if index == -1:
                break

            next_char = string_body[index + 1]
            if 'b' in modifiers.lower() and next_char in self._unicode_escapes:
                self.add_violation(
                    WrongUnicodeEscapeViolation(token, text=token.string),
                )

            # Whether it was a valid escape or not, backslash followed by
            # another character can always be consumed whole: the second
            # character can never be the start of a new backslash escape.
            index += 2 

def _check_partial_float(self, token: tokenize.TokenInfo) -> None:
        if token.string.startswith('.') or token.string.endswith('.'):
            self.add_violation(
                consistency.PartialFloatViolation(token, text=token.string),
            ) 

def _check_bad_number_suffixes(self, token: tokenize.TokenInfo) -> None:
        if self._bad_number_suffixes.match(token.string):
            self.add_violation(
                consistency.BadNumberSuffixViolation(token, text=token.string),
            )

        float_zeros = self._leading_zero_float_pattern.match(token.string)
        other_zeros = self._leading_zero_pattern.match(token.string)
        if float_zeros or other_zeros:
            self.add_violation(
                consistency.NumberWithMeaninglessZeroViolation(
                    token,
                    text=token.string,
                ),
            )

        if self._positive_exponent_pattens.match(token.string):
            self.add_violation(
                consistency.PositiveExponentViolation(
                    token,
                    text=token.string,
                ),
            )

        if token.string.startswith('0x') or token.string.startswith('0X'):
            has_wrong_hex_numbers = any(
                char in self._bad_hex_numbers
                for char in token.string
            )
            if has_wrong_hex_numbers:
                self.add_violation(
                    consistency.WrongHexNumberCaseViolation(
                        token,
                        text=token.string,
                    ),
                ) 

def _check_implicit_raw_string(self, token: tokenize.TokenInfo) -> None:
        modifiers, string_def = split_prefixes(token.string)
        if 'r' in modifiers.lower():
            return

        if self._implicit_raw_strigns.search(_replace_braces(string_def)):
            self.add_violation(
                consistency.ImplicitRawStringViolation(
                    token,
                    text=token.string,
                ),
            ) 

def _check_underscored_number(self, token: tokenize.TokenInfo) -> None:
        if '_' in token.string:
            self.add_violation(
                consistency.UnderscoredNumberViolation(
                    token,
                    text=token.string,
                ),
            ) 

def _check_complex_suffix(self, token: tokenize.TokenInfo) -> None:
        if self._bad_complex_suffix in token.string:
            self.add_violation(
                consistency.BadComplexNumberSuffixViolation(
                    token,
                    text=self._bad_complex_suffix,
                ),
            ) 

def visit_name(self, token: tokenize.TokenInfo) -> None:
        """
        Checks that ``if`` nodes are defined correctly.

        Raises:
            ImplicitElifViolation

        """
        self._check_implicit_elif(token) 

def _is_valid_shebang_line(self, token: tokenize.TokenInfo) -> bool:
        return self._shebang.match(token.line) is not None 

def _is_first_comment(self, token: tokenize.TokenInfo) -> bool:
        all_tokens = iter(self.file_tokens)
        current_token = next(all_tokens)

        while True:
            if current_token == token:
                return True
            elif current_token.exact_type not in NEWLINES:
                break
            current_token = next(all_tokens)
        return False 

def _check_cover_comments(self, token: tokenize.TokenInfo) -> None:
        comment_text = get_comment_text(token)
        match = self._no_cover.match(comment_text)
        if not match:
            return

        self._no_cover_count += 1 

def _check_correct_multiline(self, token: tokenize.TokenInfo) -> None:
        _, string_def = split_prefixes(token.string)
        if has_triple_string_quotes(string_def):
            if '\n' not in string_def and token not in self._docstrings:
                self.add_violation(
                    consistency.WrongMultilineStringViolation(token),
                ) 

def visit_comment(self, token: tokenize.TokenInfo) -> None:
        """
        Performs comment checks.

        Raises:
            OveruseOfNoqaCommentViolation
            WrongDocCommentViolation
            WrongMagicCommentViolation

        """
        self._check_noqa(token)
        self._check_typed_ast(token)
        self._check_empty_doc_comment(token)
        self._check_cover_comments(token) 

def visit(self, token: tokenize.TokenInfo) -> None:
        """
        Ensures that all string are concatenated as we allow.

        Raises:
            ImplicitStringConcatenationViolation

        """
        self._check_concatenation(token) 

def _check_concatenation(self, token: tokenize.TokenInfo) -> None:
        if token.exact_type in self._ignored_tokens:
            return

        if token.exact_type == tokenize.STRING:
            if self._previous_token:
                self.add_violation(
                    consistency.ImplicitStringConcatenationViolation(token),
                )
            self._previous_token = token
        else:
            self._previous_token = None 

def visit_name(self, token: tokenize.TokenInfo) -> None:
        """
        Check keywords related rules.

        Raises:
            MissingSpaceBetweenKeywordAndParenViolation

        """
        self._check_space_before_open_paren(token) 

def visit_dot(self, token: tokenize.TokenInfo) -> None:
        """
        Checks newline related rules.

        Raises:
            LineStartsWithDotViolation

        """
        self._check_line_starts_with_dot(token) 

def visit_any_newline(self, token: tokenize.TokenInfo) -> None:
        r"""
        Checks ``\r`` (carriage return) in line breaks.

        Raises:
            LineCompriseCarriageReturnViolation

        """
        self._check_line_comprise_carriage_return(token) 

def _check_line_starts_with_dot(self, token: tokenize.TokenInfo) -> None:
        line = token.line.lstrip()
        if line.startswith('.') and not line.startswith('...'):
            self.add_violation(LineStartsWithDotViolation(token)) 

def _check_line_comprise_carriage_return(
        self, token: tokenize.TokenInfo,
    ) -> None:
        if '\r' in token.string:
            self.add_violation(LineCompriseCarriageReturnViolation(token))