Python pygments.util.BytesIO() Examples

The following are 14 code examples of pygments.util.BytesIO(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module pygments.util , or try the search function .
Example #1
Source File: __init__.py    From pigaios with GNU General Public License v3.0 6 votes vote down vote up 
def format(tokens, formatter, outfile=None):
    """
    Format a tokenlist ``tokens`` with the formatter ``formatter``.

    If ``outfile`` is given and a valid file object (an object
    with a ``write`` method), the result will be written to it, otherwise
    it is returned as a string.
    """
    try:
        if not outfile:
            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
            formatter.format(tokens, realoutfile)
            return realoutfile.getvalue()
        else:
            formatter.format(tokens, outfile)
    except TypeError as err:
        if isinstance(err.args[0], str) and \
           ('unbound method format' in err.args[0] or
                'missing 1 required positional argument' in err.args[0]):
            raise TypeError('format() argument must be a formatter instance, '
                            'not a class')
        raise
Example #2
Source File: special.py    From pigaios with GNU General Public License v3.0 6 votes vote down vote up 
def get_tokens(self, text):
        if isinstance(text, text_type):
            # raw token stream never has any non-ASCII characters
            text = text.encode('ascii')
        if self.compress == 'gz':
            import gzip
            gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
            text = gzipfile.read()
        elif self.compress == 'bz2':
            import bz2
            text = bz2.decompress(text)

        # do not call Lexer.get_tokens() because we do not want Unicode
        # decoding to occur, and stripping is not optional.
        text = text.strip(b'\n') + b'\n'
        for i, t, v in self.get_tokens_unprocessed(text):
            yield t, v
Example #3
Source File: __init__.py    From pySINDy with MIT License 6 votes vote down vote up 
def format(tokens, formatter, outfile=None):  # pylint: disable=redefined-builtin
    """
    Format a tokenlist ``tokens`` with the formatter ``formatter``.

    If ``outfile`` is given and a valid file object (an object
    with a ``write`` method), the result will be written to it, otherwise
    it is returned as a string.
    """
    try:
        if not outfile:
            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
            formatter.format(tokens, realoutfile)
            return realoutfile.getvalue()
        else:
            formatter.format(tokens, outfile)
    except TypeError as err:
        if (isinstance(err.args[0], str) and
            ('unbound method format' in err.args[0] or
             'missing 1 required positional argument' in err.args[0])):
            raise TypeError('format() argument must be a formatter instance, '
                            'not a class')
        raise
Example #4
Source File: special.py    From pySINDy with MIT License 6 votes vote down vote up 
def get_tokens(self, text):
        if isinstance(text, text_type):
            # raw token stream never has any non-ASCII characters
            text = text.encode('ascii')
        if self.compress == 'gz':
            import gzip
            gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
            text = gzipfile.read()
        elif self.compress == 'bz2':
            import bz2
            text = bz2.decompress(text)

        # do not call Lexer.get_tokens() because we do not want Unicode
        # decoding to occur, and stripping is not optional.
        text = text.strip(b'\n') + b'\n'
        for i, t, v in self.get_tokens_unprocessed(text):
            yield t, v
Example #5
Source File: __init__.py    From komodo-wakatime with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
def format(tokens, formatter, outfile=None):  # pylint: disable=redefined-builtin
    """
    Format a tokenlist ``tokens`` with the formatter ``formatter``.

    If ``outfile`` is given and a valid file object (an object
    with a ``write`` method), the result will be written to it, otherwise
    it is returned as a string.
    """
    try:
        if not outfile:
            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
            formatter.format(tokens, realoutfile)
            return realoutfile.getvalue()
        else:
            formatter.format(tokens, outfile)
    except TypeError as err:
        if (isinstance(err.args[0], str) and
            ('unbound method format' in err.args[0] or
             'missing 1 required positional argument' in err.args[0])):
            raise TypeError('format() argument must be a formatter instance, '
                            'not a class')
        raise
Example #6
Source File: special.py    From komodo-wakatime with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
def get_tokens(self, text):
        if isinstance(text, text_type):
            # raw token stream never has any non-ASCII characters
            text = text.encode('ascii')
        if self.compress == 'gz':
            import gzip
            gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
            text = gzipfile.read()
        elif self.compress == 'bz2':
            import bz2
            text = bz2.decompress(text)

        # do not call Lexer.get_tokens() because we do not want Unicode
        # decoding to occur, and stripping is not optional.
        text = text.strip(b'\n') + b'\n'
        for i, t, v in self.get_tokens_unprocessed(text):
            yield t, v
Example #7
Source File: __init__.py    From Carnets with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
def format(tokens, formatter, outfile=None):  # pylint: disable=redefined-builtin
    """
    Format a tokenlist ``tokens`` with the formatter ``formatter``.

    If ``outfile`` is given and a valid file object (an object
    with a ``write`` method), the result will be written to it, otherwise
    it is returned as a string.
    """
    try:
        if not outfile:
            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
            formatter.format(tokens, realoutfile)
            return realoutfile.getvalue()
        else:
            formatter.format(tokens, outfile)
    except TypeError as err:
        if (isinstance(err.args[0], str) and
            ('unbound method format' in err.args[0] or
             'missing 1 required positional argument' in err.args[0])):
            raise TypeError('format() argument must be a formatter instance, '
                            'not a class')
        raise
Example #8
Source File: special.py    From Carnets with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
def get_tokens(self, text):
        if isinstance(text, text_type):
            # raw token stream never has any non-ASCII characters
            text = text.encode('ascii')
        if self.compress == 'gz':
            import gzip
            gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
            text = gzipfile.read()
        elif self.compress == 'bz2':
            import bz2
            text = bz2.decompress(text)

        # do not call Lexer.get_tokens() because we do not want Unicode
        # decoding to occur, and stripping is not optional.
        text = text.strip(b'\n') + b'\n'
        for i, t, v in self.get_tokens_unprocessed(text):
            yield t, v
Example #9
Source File: __init__.py    From diaphora with GNU Affero General Public License v3.0 6 votes vote down vote up 
def format(tokens, formatter, outfile=None):
    """
    Format a tokenlist ``tokens`` with the formatter ``formatter``.

    If ``outfile`` is given and a valid file object (an object
    with a ``write`` method), the result will be written to it, otherwise
    it is returned as a string.
    """
    try:
        if not outfile:
            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
            formatter.format(tokens, realoutfile)
            return realoutfile.getvalue()
        else:
            formatter.format(tokens, outfile)
    except TypeError as err:
        if isinstance(err.args[0], str) and \
           ('unbound method format' in err.args[0] or
                'missing 1 required positional argument' in err.args[0]):
            raise TypeError('format() argument must be a formatter instance, '
                            'not a class')
        raise
Example #10
Source File: special.py    From diaphora with GNU Affero General Public License v3.0 6 votes vote down vote up 
def get_tokens(self, text):
        if isinstance(text, text_type):
            # raw token stream never has any non-ASCII characters
            text = text.encode('ascii')
        if self.compress == 'gz':
            import gzip
            gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
            text = gzipfile.read()
        elif self.compress == 'bz2':
            import bz2
            text = bz2.decompress(text)

        # do not call Lexer.get_tokens() because we do not want Unicode
        # decoding to occur, and stripping is not optional.
        text = text.strip(b'\n') + b'\n'
        for i, t, v in self.get_tokens_unprocessed(text):
            yield t, v
Example #11
Source File: __init__.py    From android_universal with MIT License 6 votes vote down vote up 
def format(tokens, formatter, outfile=None):  # pylint: disable=redefined-builtin
    """
    Format a tokenlist ``tokens`` with the formatter ``formatter``.

    If ``outfile`` is given and a valid file object (an object
    with a ``write`` method), the result will be written to it, otherwise
    it is returned as a string.
    """
    try:
        if not outfile:
            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
            formatter.format(tokens, realoutfile)
            return realoutfile.getvalue()
        else:
            formatter.format(tokens, outfile)
    except TypeError as err:
        if (isinstance(err.args[0], str) and
            ('unbound method format' in err.args[0] or
             'missing 1 required positional argument' in err.args[0])):
            raise TypeError('format() argument must be a formatter instance, '
                            'not a class')
        raise
Example #12
Source File: special.py    From android_universal with MIT License 6 votes vote down vote up 
def get_tokens(self, text):
        if isinstance(text, text_type):
            # raw token stream never has any non-ASCII characters
            text = text.encode('ascii')
        if self.compress == 'gz':
            import gzip
            gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
            text = gzipfile.read()
        elif self.compress == 'bz2':
            import bz2
            text = bz2.decompress(text)

        # do not call Lexer.get_tokens() because we do not want Unicode
        # decoding to occur, and stripping is not optional.
        text = text.strip(b'\n') + b'\n'
        for i, t, v in self.get_tokens_unprocessed(text):
            yield t, v
Example #13
Source File: __init__.py    From syntax-highlighting with GNU Affero General Public License v3.0 6 votes vote down vote up 
def format(tokens, formatter, outfile=None):  # pylint: disable=redefined-builtin
    """
    Format a tokenlist ``tokens`` with the formatter ``formatter``.

    If ``outfile`` is given and a valid file object (an object
    with a ``write`` method), the result will be written to it, otherwise
    it is returned as a string.
    """
    try:
        if not outfile:
            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
            formatter.format(tokens, realoutfile)
            return realoutfile.getvalue()
        else:
            formatter.format(tokens, outfile)
    except TypeError as err:
        if (isinstance(err.args[0], str) and
            ('unbound method format' in err.args[0] or
             'missing 1 required positional argument' in err.args[0])):
            raise TypeError('format() argument must be a formatter instance, '
                            'not a class')
        raise
Example #14
Source File: special.py    From syntax-highlighting with GNU Affero General Public License v3.0 6 votes vote down vote up 
def get_tokens(self, text):
        if isinstance(text, text_type):
            # raw token stream never has any non-ASCII characters
            text = text.encode('ascii')
        if self.compress == 'gz':
            import gzip
            gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
            text = gzipfile.read()
        elif self.compress == 'bz2':
            import bz2
            text = bz2.decompress(text)

        # do not call Lexer.get_tokens() because we do not want Unicode
        # decoding to occur, and stripping is not optional.
        text = text.strip(b'\n') + b'\n'
        for i, t, v in self.get_tokens_unprocessed(text):
            yield t, v