Python tokenize.untokenize() Examples
The following are 30
code examples of tokenize.untokenize().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
tokenize
, or try the search function
.
Example #1
Source File: tilde.py From panoramix with MIT License | 7 votes |
def make_exp(exp): check_type, out_name, concrete = None, None, None if len(exp) == 1 and exp[0][1] == "_": return (None, None, None) if len(exp) == 2 and exp[0][1] == "*" and exp[1][0] == NAME: exp = [(OP, "..."), (OP, ":"), exp[1]] if exp[0][1] in ("...", "int", "str", "list", "tuple"): check_type = exp[0][1] exp.pop(0) if len(exp) == 2 and exp[0][1] == ":": out_name = exp[1][1] elif len(exp) > 0: concrete = tokenize.untokenize(exp) # .decode() return (check_type, out_name, concrete)
Example #2
Source File: vfp2py.py From vfp2py with MIT License | 6 votes |
def prg2py_after_preproc(data, parser_start, input_filename): input_stream = antlr4.InputStream(data) lexer = VisualFoxpro9Lexer(input_stream) stream = antlr4.CommonTokenStream(lexer) parser = VisualFoxpro9Parser(stream) tree = run_parser(stream, parser, parser_start) TreeCleanVisitor().visit(tree) output_tree = PythonConvertVisitor(input_filename).visit(tree) if not isinstance(output_tree, list): return output_tree output = add_indents(output_tree, 0) options = autopep8.parse_args(['--max-line-length', '100000', '-']) output = autopep8.fix_code(output, options) tokens = list(tokenize.generate_tokens(io.StringIO(output).readline)) for i, token in enumerate(tokens): token = list(token) if token[0] == tokenize.STRING and token[1].startswith('u'): token[1] = token[1][1:] tokens[i] = tuple(token) return tokenize.untokenize(tokens)
Example #3
Source File: minify.py From mitogen with BSD 3-Clause "New" or "Revised" License | 6 votes |
def minimize_source(source): """ Remove comments and docstrings from Python `source`, preserving line numbers and syntax of empty blocks. :param str source: The source to minimize. :returns str: The minimized source. """ source = mitogen.core.to_text(source) tokens = tokenize.generate_tokens(StringIO(source).readline) tokens = strip_comments(tokens) tokens = strip_docstrings(tokens) tokens = reindent(tokens) return tokenize.untokenize(tokens)
Example #4
Source File: reader.py From py2nb with BSD 3-Clause "New" or "Revised" License | 6 votes |
def read(filename): """ Read a regular Python file with special formatting and performance preprocessing on it. The result is a string that conforms to the IPython notebook version 3 python script format. """ with open(filename, 'rb') as fin: token_gen = _generate_tokens(fin.readline) cvt_docstr_gen = convert_toplevel_docstring(token_gen) nl_gen = fix_newlines(cvt_docstr_gen) out = list(nl_gen) formatted = tokenize.untokenize(out).decode('utf-8') return fix_empty_lines(formatted) # ============================================================================= # Helpers # =============================================================================
Example #5
Source File: __init__.py From accloudtant with Apache License 2.0 | 5 votes |
def fix_lazy_json(in_text): """ This function modifies JS-contained JSON to be valid. Posted in http://stackoverflow.com/questions/4033633/handling-lazy-json-\ in-python-expecting-property-name by Pau Sánchez (codigomanso.com) """ tokengen = tokenize.generate_tokens(io.StringIO(in_text).readline) valid_tokens = ['true', 'false', 'null', '-Infinity', 'Infinity', 'NaN'] result = [] for tokid, tokval, _, _, _ in tokengen: # fix unquoted strings if tokid == token.NAME: tokid, tokval = fix_unquoted((tokid, tokval), valid_tokens) # fix single-quoted strings elif tokid == token.STRING: tokval = fix_single_quoted(tokval) # remove invalid commas elif (tokid == token.OP) and ((tokval == '}') or (tokval == ']')): result = remove_invalid_commas(result) result.append((tokid, tokval)) return tokenize.untokenize(result)
Example #6
Source File: format.py From mxnet-lambda with Apache License 2.0 | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False for token in tokenize.generate_tokens(StringIO(asstr(s)).read): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) return tokenize.untokenize(tokens)
Example #7
Source File: retype_hgext.py From retype with MIT License | 5 votes |
def apply_job_security(code): """Treat input `code` like Python 2 (implicit strings are byte literals). The implementation is horribly inefficient but the goal is to be compatible with what Mercurial does at runtime. """ buf = io.BytesIO(code.encode("utf8")) tokens = tokenize.tokenize(buf.readline) # NOTE: by setting the fullname to `mercurial.pycompat` below, we're # ensuring that hg-specific pycompat imports aren't inserted to the code. data = tokenize.untokenize(replacetokens(list(tokens), "mercurial.pycompat")) return cast(str, data.decode("utf8"))
Example #8
Source File: token_utils.py From pyminifier with GNU General Public License v3.0 | 5 votes |
def untokenize(tokens): """ Converts the output of tokenize.generate_tokens back into a human-readable string (that doesn't contain oddly-placed whitespace everywhere). .. note:: Unlike :meth:`tokenize.untokenize`, this function requires the 3rd and 4th items in each token tuple (though we can use lists *or* tuples). """ out = "" last_lineno = -1 last_col = 0 for tok in tokens: token_string = tok[1] start_line, start_col = tok[2] end_line, end_col = tok[3] # The following two conditionals preserve indentation: if start_line > last_lineno: last_col = 0 if start_col > last_col and token_string != '\n': out += (" " * (start_col - last_col)) out += token_string last_col = end_col last_lineno = end_line return out
Example #9
Source File: GoogleFinanceQuoteAdapter.py From paperbroker with MIT License | 5 votes |
def fixLazyJson (in_text): tokengen = tokenize.generate_tokens(StringIO(in_text.decode('ascii')).readline) result = [] for tokid, tokval, _, _, _ in tokengen: # fix unquoted strings if (tokid == token.NAME): if tokval not in ['true', 'false', 'null', '-Infinity', 'Infinity', 'NaN']: tokid = token.STRING tokval = u'"%s"' % tokval # fix single-quoted strings elif (tokid == token.STRING): if tokval.startswith ("'"): tokval = u'"%s"' % tokval[1:-1].replace ('"', '\\"') # remove invalid commas elif (tokid == token.OP) and ((tokval == '}') or (tokval == ']')): if (len(result) > 0) and (result[-1][1] == ','): result.pop() # fix single-quoted strings elif (tokid == token.STRING): if tokval.startswith ("'"): tokval = u'"%s"' % tokval[1:-1].replace ('"', '\\"') result.append((tokid, tokval)) return tokenize.untokenize(result)
Example #10
Source File: format.py From ImageFusion with MIT License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False for token in tokenize.generate_tokens(StringIO(asstr(s)).read): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) return tokenize.untokenize(tokens)
Example #11
Source File: format.py From Splunking-Crime with GNU Affero General Public License v3.0 | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False for token in tokenize.generate_tokens(StringIO(asstr(s)).read): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) return tokenize.untokenize(tokens)
Example #12
Source File: expr.py From Splunking-Crime with GNU Affero General Public License v3.0 | 5 votes |
def _preparse(source, f=compose(_replace_locals, _replace_booleans, _rewrite_assign)): """Compose a collection of tokenization functions Parameters ---------- source : str A Python source code string f : callable This takes a tuple of (toknum, tokval) as its argument and returns a tuple with the same structure but possibly different elements. Defaults to the composition of ``_rewrite_assign``, ``_replace_booleans``, and ``_replace_locals``. Returns ------- s : str Valid Python source code Notes ----- The `f` parameter can be any callable that takes *and* returns input of the form ``(toknum, tokval)``, where ``toknum`` is one of the constants from the ``tokenize`` module and ``tokval`` is a string. """ assert callable(f), 'f must be callable' return tokenize.untokenize(lmap(f, tokenize_string(source)))
Example #13
Source File: format.py From elasticintel with GNU General Public License v3.0 | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False for token in tokenize.generate_tokens(StringIO(asstr(s)).read): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) return tokenize.untokenize(tokens)
Example #14
Source File: expr.py From elasticintel with GNU General Public License v3.0 | 5 votes |
def _preparse(source, f=compose(_replace_locals, _replace_booleans, _rewrite_assign)): """Compose a collection of tokenization functions Parameters ---------- source : str A Python source code string f : callable This takes a tuple of (toknum, tokval) as its argument and returns a tuple with the same structure but possibly different elements. Defaults to the composition of ``_rewrite_assign``, ``_replace_booleans``, and ``_replace_locals``. Returns ------- s : str Valid Python source code Notes ----- The `f` parameter can be any callable that takes *and* returns input of the form ``(toknum, tokval)``, where ``toknum`` is one of the constants from the ``tokenize`` module and ``tokval`` is a string. """ assert callable(f), 'f must be callable' return tokenize.untokenize(lmap(f, tokenize_string(source)))
Example #15
Source File: format.py From recruit with Apache License 2.0 | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False # adding newline as python 2.7.5 workaround string = asstr(s) + "\n" for token in tokenize.generate_tokens(StringIO(string).readline): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) # removing newline (see above) as python 2.7.5 workaround return tokenize.untokenize(tokens)[:-1]
Example #16
Source File: format.py From coffeegrindsize with MIT License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False # adding newline as python 2.7.5 workaround string = asstr(s) + "\n" for token in tokenize.generate_tokens(StringIO(string).readline): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) # removing newline (see above) as python 2.7.5 workaround return tokenize.untokenize(tokens)[:-1]
Example #17
Source File: tilde.py From panoramix with MIT License | 5 votes |
def untilde(x): return str( tokenize.untokenize(translate(BytesIO(x.encode("utf-8")).readline)).decode() )
Example #18
Source File: tilde.py From panoramix with MIT License | 5 votes |
def __init__(self, *args, **kwargs): codecs.StreamReader.__init__(self, *args, **kwargs) data = tokenize.untokenize(translate(self.stream.readline)) self.stream = i.StringIO(data)
Example #19
Source File: format.py From Carnets with BSD 3-Clause "New" or "Revised" License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False # adding newline as python 2.7.5 workaround string = asstr(s) + "\n" for token in tokenize.generate_tokens(StringIO(string).readline): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) # removing newline (see above) as python 2.7.5 workaround return tokenize.untokenize(tokens)[:-1]
Example #20
Source File: parser.py From lax with GNU General Public License v2.0 | 5 votes |
def build(data): result = [] for num, val, _, _, _ in split(StringIO(data).readline): if num == NUMBER: result.extend([(NAME, 'Num'), (OP, '('), (STRING, str(val)), (OP, ')')]) elif num == NAME and not val.startswith('_'): result.extend([(NAME, 'Chk'), (OP, '('), (OP, "'"), (STRING, str(val)), (OP, "'"), (OP, ')')]) else: result.append((num, val)) return untokenize(result)
Example #21
Source File: format.py From Serverless-Deep-Learning-with-TensorFlow-and-AWS-Lambda with MIT License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False # adding newline as python 2.7.5 workaround string = asstr(s) + "\n" for token in tokenize.generate_tokens(StringIO(string).readline): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) # removing newline (see above) as python 2.7.5 workaround return tokenize.untokenize(tokens)[:-1]
Example #22
Source File: format.py From twitter-stock-recommendation with MIT License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False # adding newline as python 2.7.5 workaround string = asstr(s) + "\n" for token in tokenize.generate_tokens(StringIO(string).readline): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) # removing newline (see above) as python 2.7.5 workaround return tokenize.untokenize(tokens)[:-1]
Example #23
Source File: format.py From keras-lambda with MIT License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False for token in tokenize.generate_tokens(StringIO(asstr(s)).read): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) return tokenize.untokenize(tokens)
Example #24
Source File: format.py From GraphicDesignPatternByPython with MIT License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False # adding newline as python 2.7.5 workaround string = asstr(s) + "\n" for token in tokenize.generate_tokens(StringIO(string).readline): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) # removing newline (see above) as python 2.7.5 workaround return tokenize.untokenize(tokens)[:-1]
Example #25
Source File: format.py From lambda-packs with MIT License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False # adding newline as python 2.7.5 workaround string = asstr(s) + "\n" for token in tokenize.generate_tokens(StringIO(string).readline): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) # removing newline (see above) as python 2.7.5 workaround return tokenize.untokenize(tokens)[:-1]
Example #26
Source File: format.py From lambda-packs with MIT License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False for token in tokenize.generate_tokens(StringIO(asstr(s)).read): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) return tokenize.untokenize(tokens)
Example #27
Source File: format.py From auto-alt-text-lambda-api with MIT License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False for token in tokenize.generate_tokens(StringIO(asstr(s)).read): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) return tokenize.untokenize(tokens)
Example #28
Source File: format.py From vnpy_crypto with MIT License | 5 votes |
def _filter_header(s): """Clean up 'L' in npz header ints. Cleans up the 'L' in strings representing integers. Needed to allow npz headers produced in Python2 to be read in Python3. Parameters ---------- s : byte string Npy file header. Returns ------- header : str Cleaned up header. """ import tokenize if sys.version_info[0] >= 3: from io import StringIO else: from StringIO import StringIO tokens = [] last_token_was_number = False # adding newline as python 2.7.5 workaround string = asstr(s) + "\n" for token in tokenize.generate_tokens(StringIO(string).readline): token_type = token[0] token_string = token[1] if (last_token_was_number and token_type == tokenize.NAME and token_string == "L"): continue else: tokens.append(token) last_token_was_number = (token_type == tokenize.NUMBER) # removing newline (see above) as python 2.7.5 workaround return tokenize.untokenize(tokens)[:-1]
Example #29
Source File: expr.py From vnpy_crypto with MIT License | 5 votes |
def _preparse(source, f=compose(_replace_locals, _replace_booleans, _rewrite_assign)): """Compose a collection of tokenization functions Parameters ---------- source : str A Python source code string f : callable This takes a tuple of (toknum, tokval) as its argument and returns a tuple with the same structure but possibly different elements. Defaults to the composition of ``_rewrite_assign``, ``_replace_booleans``, and ``_replace_locals``. Returns ------- s : str Valid Python source code Notes ----- The `f` parameter can be any callable that takes *and* returns input of the form ``(toknum, tokval)``, where ``toknum`` is one of the constants from the ``tokenize`` module and ``tokval`` is a string. """ assert callable(f), 'f must be callable' return tokenize.untokenize(lmap(f, tokenize_string(source)))
Example #30
Source File: expr.py From Computable with MIT License | 5 votes |
def _rewrite_assign(source): """Rewrite the assignment operator for PyTables expression that want to use ``=`` as a substitute for ``==``. """ res = [] g = tokenize.generate_tokens(StringIO(source).readline) for toknum, tokval, _, _, _ in g: res.append((toknum, '==' if tokval == '=' else tokval)) return tokenize.untokenize(res)