Python codecs.utf_16_le_decode() Examples

The following are 30 code examples of codecs.utf_16_le_decode(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module codecs , or try the search function .
Example #1
Source File: test_codecs.py    From ironpython2 with Apache License 2.0 6 votes vote down vote up 
def test_errors(self):
        tests = [
            (b'\xff', u'\ufffd'),
            (b'A\x00Z', u'A\ufffd'),
            (b'A\x00B\x00C\x00D\x00Z', u'ABCD\ufffd'),
            (b'\x00\xd8', u'\ufffd'),
            (b'\x00\xd8A', u'\ufffd'),
            (b'\x00\xd8A\x00', u'\ufffdA'),
            (b'\x00\xdcA\x00', u'\ufffdA'),
        ]
        for raw, expected in tests:
            try:
                with self.assertRaises(UnicodeDecodeError):
                    codecs.utf_16_le_decode(raw, 'strict', True)
                self.assertEqual(raw.decode('utf-16le', 'replace'), expected)
            except:
                print 'raw=%r' % raw
                raise
Example #2
Source File: test_codecs.py    From ironpython3 with Apache License 2.0 6 votes vote down vote up 
def test_errors(self):
        tests = [
            (b'\xff', '\ufffd'),
            (b'A\x00Z', 'A\ufffd'),
            (b'A\x00B\x00C\x00D\x00Z', 'ABCD\ufffd'),
            (b'\x00\xd8', '\ufffd'),
            (b'\x00\xd8A', '\ufffd'),
            (b'\x00\xd8A\x00', '\ufffdA'),
            (b'\x00\xdcA\x00', '\ufffdA'),
        ]
        if sys.implementation.name == 'ironpython':
            # IronPython's strings are in the UTF-16 form thus decoding
            # a surrogate pair always results in two characters. Consequently,
            # decoding a broken surrogate pair results in TWO U+FFFD
            tests[4] = (b'\x00\xd8A', '\ufffd\ufffd')
        for raw, expected in tests:
            self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode,
                              raw, 'strict', True)
            self.assertEqual(raw.decode('utf-16le', 'replace'), expected)
Example #3
Source File: utf_16.py    From Imogen with MIT License 5 votes vote down vote up 
def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError("UTF-16 stream does not start with BOM")
        return (object, consumed)

### encodings module API
Example #4
Source File: utf_16.py    From ImageFusion with MIT License 5 votes vote down vote up 
def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)
Example #5
Source File: utf_16.py    From ImageFusion with MIT License 5 votes vote down vote up 
def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError,"UTF-16 stream does not start with BOM"
        return (object, consumed)

### encodings module API
Example #6
Source File: utf_16_le.py    From ImageFusion with MIT License 5 votes vote down vote up 
def decode(input, errors='strict'):
    return codecs.utf_16_le_decode(input, errors, True)
Example #7
Source File: reader.py    From luscan-devel with GNU General Public License v2.0 5 votes vote down vote up 
def determine_encoding(self):
        while not self.eof and len(self.raw_buffer) < 2:
            self.update_raw()
        if not isinstance(self.raw_buffer, unicode):
            if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
                self.raw_decode = codecs.utf_16_le_decode
                self.encoding = 'utf-16-le'
            elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
                self.raw_decode = codecs.utf_16_be_decode
                self.encoding = 'utf-16-be'
            else:
                self.raw_decode = codecs.utf_8_decode
                self.encoding = 'utf-8'
        self.update(1)
Example #8
Source File: utf_16.py    From python with Apache License 2.0 5 votes vote down vote up 
def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError("UTF-16 stream does not start with BOM")
        return (object, consumed)

### encodings module API
Example #9
Source File: utf_16.py    From python with Apache License 2.0 5 votes vote down vote up 
def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None
Example #10
Source File: utf_16.py    From python with Apache License 2.0 5 votes vote down vote up 
def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)
Example #11
Source File: reader.py    From opsbro with MIT License 5 votes vote down vote up 
def determine_encoding(self):
        while not self.eof and (self.raw_buffer is None or
                                len(self.raw_buffer) < 2):
            self.update_raw()
        if isinstance(self.raw_buffer, binary_type):
            if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
                self.raw_decode = codecs.utf_16_le_decode
                self.encoding = 'utf-16-le'
            elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
                self.raw_decode = codecs.utf_16_be_decode
                self.encoding = 'utf-16-be'
            else:
                self.raw_decode = codecs.utf_8_decode
                self.encoding = 'utf-8'
        self.update(1)
Example #12
Source File: utf_16.py    From telegram-robot-rss with Mozilla Public License 2.0 5 votes vote down vote up 
def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)
Example #13
Source File: utf_16_le.py    From telegram-robot-rss with Mozilla Public License 2.0 5 votes vote down vote up 
def decode(input, errors='strict'):
    return codecs.utf_16_le_decode(input, errors, True)
Example #14
Source File: utf_16.py    From scylla with Apache License 2.0 5 votes vote down vote up 
def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError("UTF-16 stream does not start with BOM")
        return (object, consumed)

### encodings module API
Example #15
Source File: utf_16.py    From scylla with Apache License 2.0 5 votes vote down vote up 
def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None
Example #16
Source File: utf_16.py    From scylla with Apache License 2.0 5 votes vote down vote up 
def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)
Example #17
Source File: utf_16_le.py    From scylla with Apache License 2.0 5 votes vote down vote up 
def decode(input, errors='strict'):
    return codecs.utf_16_le_decode(input, errors, True)
Example #18
Source File: reader.py    From cronyo with MIT License 5 votes vote down vote up 
def determine_encoding(self):
        while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2):
            self.update_raw()
        if isinstance(self.raw_buffer, bytes):
            if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
                self.raw_decode = codecs.utf_16_le_decode
                self.encoding = 'utf-16-le'
            elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
                self.raw_decode = codecs.utf_16_be_decode
                self.encoding = 'utf-16-be'
            else:
                self.raw_decode = codecs.utf_8_decode
                self.encoding = 'utf-8'
        self.update(1)
Example #19
Source File: utf_16.py    From Imogen with MIT License 5 votes vote down vote up 
def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None
Example #20
Source File: utf_16.py    From Imogen with MIT License 5 votes vote down vote up 
def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)
Example #21
Source File: utf_16_le.py    From Imogen with MIT License 5 votes vote down vote up 
def decode(input, errors='strict'):
    return codecs.utf_16_le_decode(input, errors, True)
Example #22
Source File: test_codecs.py    From Fluid-Designer with GNU General Public License v3.0 5 votes vote down vote up 
def test_errors(self):
        tests = [
            (b'\xff', '\ufffd'),
            (b'A\x00Z', 'A\ufffd'),
            (b'A\x00B\x00C\x00D\x00Z', 'ABCD\ufffd'),
            (b'\x00\xd8', '\ufffd'),
            (b'\x00\xd8A', '\ufffd'),
            (b'\x00\xd8A\x00', '\ufffdA'),
            (b'\x00\xdcA\x00', '\ufffdA'),
        ]
        for raw, expected in tests:
            self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode,
                              raw, 'strict', True)
            self.assertEqual(raw.decode('utf-16le', 'replace'), expected)
Example #23
Source File: utf_16.py    From Fluid-Designer with GNU General Public License v3.0 5 votes vote down vote up 
def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError("UTF-16 stream does not start with BOM")
        return (object, consumed)

### encodings module API
Example #24
Source File: utf_16.py    From Fluid-Designer with GNU General Public License v3.0 5 votes vote down vote up 
def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None
Example #25
Source File: utf_16.py    From Fluid-Designer with GNU General Public License v3.0 5 votes vote down vote up 
def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)
Example #26
Source File: utf_16_le.py    From Fluid-Designer with GNU General Public License v3.0 5 votes vote down vote up 
def decode(input, errors='strict'):
    return codecs.utf_16_le_decode(input, errors, True)
Example #27
Source File: reader.py    From GraphicDesignPatternByPython with MIT License 5 votes vote down vote up 
def determine_encoding(self):
        while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2):
            self.update_raw()
        if isinstance(self.raw_buffer, bytes):
            if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
                self.raw_decode = codecs.utf_16_le_decode
                self.encoding = 'utf-16-le'
            elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
                self.raw_decode = codecs.utf_16_be_decode
                self.encoding = 'utf-16-be'
            else:
                self.raw_decode = codecs.utf_8_decode
                self.encoding = 'utf-8'
        self.update(1)
Example #28
Source File: utf_16.py    From GraphicDesignPatternByPython with MIT License 5 votes vote down vote up 
def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError("UTF-16 stream does not start with BOM")
        return (object, consumed)

### encodings module API
Example #29
Source File: utf_16.py    From GraphicDesignPatternByPython with MIT License 5 votes vote down vote up 
def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None
Example #30
Source File: utf_16.py    From GraphicDesignPatternByPython with MIT License 5 votes vote down vote up 
def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)