Python codecs.utf_16_be_decode() Examples
The following are 30
code examples of codecs.utf_16_be_decode().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
codecs
, or try the search function
.
.
Example #1
| Source File: test_codecs.py From Fluid-Designer with GNU General Public License v3.0 | 6 votes |
|
def test_decode_unicode(self):
# Most decoders don't accept unicode input
decoders = [
codecs.utf_7_decode,
codecs.utf_8_decode,
codecs.utf_16_le_decode,
codecs.utf_16_be_decode,
codecs.utf_16_ex_decode,
codecs.utf_32_decode,
codecs.utf_32_le_decode,
codecs.utf_32_be_decode,
codecs.utf_32_ex_decode,
codecs.latin_1_decode,
codecs.ascii_decode,
codecs.charmap_decode,
]
if hasattr(codecs, "mbcs_decode"):
decoders.append(codecs.mbcs_decode)
for decoder in decoders:
self.assertRaises(TypeError, decoder, "xxx")
Example #2
| Source File: test_codecs.py From ironpython3 with Apache License 2.0 | 6 votes |
|
def test_decode_unicode(self):
# Most decoders don't accept unicode input
decoders = [
codecs.utf_7_decode,
codecs.utf_8_decode,
codecs.utf_16_le_decode,
codecs.utf_16_be_decode,
codecs.utf_16_ex_decode,
codecs.utf_32_decode,
codecs.utf_32_le_decode,
codecs.utf_32_be_decode,
codecs.utf_32_ex_decode,
codecs.latin_1_decode,
codecs.ascii_decode,
codecs.charmap_decode,
]
if hasattr(codecs, "mbcs_decode"):
decoders.append(codecs.mbcs_decode)
for decoder in decoders:
self.assertRaises(TypeError, decoder, "xxx")
Example #3
| Source File: test_codecs.py From ironpython2 with Apache License 2.0 | 6 votes |
|
def test_errors(self):
tests = [
(b'\xff', u'\ufffd'),
(b'\x00A\xff', u'A\ufffd'),
(b'\x00A\x00B\x00C\x00DZ', u'ABCD\ufffd'),
(b'\xd8\x00', u'\ufffd'),
(b'\xd8\x00\xdc', u'\ufffd'),
(b'\xd8\x00\x00A', u'\ufffdA'),
(b'\xdc\x00\x00A', u'\ufffdA'),
]
for raw, expected in tests:
try:
with self.assertRaises(UnicodeDecodeError):
codecs.utf_16_be_decode(raw, 'strict', True)
self.assertEqual(raw.decode('utf-16be', 'replace'), expected)
except:
print 'raw=%r' % raw
raise
Example #4
| Source File: utf_16.py From ironpython3 with Apache License 2.0 | 5 votes |
|
def setstate(self, state):
# state[1] will be ignored by BufferedIncrementalDecoder.setstate()
codecs.BufferedIncrementalDecoder.setstate(self, state)
state = state[1]
if state == 0:
self.decoder = (codecs.utf_16_be_decode
if sys.byteorder == "big"
else codecs.utf_16_le_decode)
elif state == 1:
self.decoder = (codecs.utf_16_le_decode
if sys.byteorder == "big"
else codecs.utf_16_be_decode)
else:
self.decoder = None
Example #5
| Source File: utf_16.py From Imogen with MIT License | 5 votes |
|
def setstate(self, state):
# state[1] will be ignored by BufferedIncrementalDecoder.setstate()
codecs.BufferedIncrementalDecoder.setstate(self, state)
state = state[1]
if state == 0:
self.decoder = (codecs.utf_16_be_decode
if sys.byteorder == "big"
else codecs.utf_16_le_decode)
elif state == 1:
self.decoder = (codecs.utf_16_le_decode
if sys.byteorder == "big"
else codecs.utf_16_be_decode)
else:
self.decoder = None
Example #6
| Source File: utf_16.py From Imogen with MIT License | 5 votes |
|
def decode(self, input, errors='strict'):
(object, consumed, byteorder) = \
codecs.utf_16_ex_decode(input, errors, 0, False)
if byteorder == -1:
self.decode = codecs.utf_16_le_decode
elif byteorder == 1:
self.decode = codecs.utf_16_be_decode
elif consumed>=2:
raise UnicodeError("UTF-16 stream does not start with BOM")
return (object, consumed)
### encodings module API
Example #7
| Source File: utf_16_be.py From Imogen with MIT License | 5 votes |
|
def decode(input, errors='strict'):
return codecs.utf_16_be_decode(input, errors, True)
Example #8
| Source File: utf_16_be.py From scylla with Apache License 2.0 | 5 votes |
|
def decode(input, errors='strict'):
return codecs.utf_16_be_decode(input, errors, True)
Example #9
| Source File: utf_16.py From scylla with Apache License 2.0 | 5 votes |
|
def _buffer_decode(self, input, errors, final):
if self.decoder is None:
(output, consumed, byteorder) = \
codecs.utf_16_ex_decode(input, errors, 0, final)
if byteorder == -1:
self.decoder = codecs.utf_16_le_decode
elif byteorder == 1:
self.decoder = codecs.utf_16_be_decode
elif consumed >= 2:
raise UnicodeError("UTF-16 stream does not start with BOM")
return (output, consumed)
return self.decoder(input, self.errors, final)
Example #10
| Source File: test_codecs.py From ironpython3 with Apache License 2.0 | 5 votes |
|
def test_utf_16_be_decode_incremental(self):
b = b"\xff\xfe\x00\x41\xd9\x00\xdd\x00\xdc\x00\xd8\x00\xdc\x00"
expected = [
('', 0),
('', 0),
('\ufffe', 2),
('\ufffe', 2),
('\ufffeA', 4),
('\ufffeA', 4),
('\ufffeA', 4),
('\ufffeA', 4),
('\ufffeA\U00050100', 8),
('\ufffeA\U00050100', 8),
('\ufffeA\U00050100\ufffd', 10),
('\ufffeA\U00050100\ufffd', 10),
('\ufffeA\U00050100\ufffd', 10),
('\ufffeA\U00050100\ufffd', 10),
('\ufffeA\U00050100\ufffd\U00010000', 14)
]
if not is_cli:
# CPython's strings are UTF-32 so an invalid surrogate pair results in one replacement char.
# Therefore CPython cannot report error on a dangling low surrogate until it verifies
# that the next char is not an invalid surrogate as well.
expected[10] = expected[11] = ('\ufffeA\U00050100', 8)
for i in range(len(b) + 1):
res = codecs.utf_16_be_decode(b[:i], 'replace')
self.assertEqual(res, expected[i])
self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode, b"\x41\x00\xd8\x00\xd8\x00", 'strict', False)
Example #11
| Source File: utf_16.py From Imogen with MIT License | 5 votes |
|
def _buffer_decode(self, input, errors, final):
if self.decoder is None:
(output, consumed, byteorder) = \
codecs.utf_16_ex_decode(input, errors, 0, final)
if byteorder == -1:
self.decoder = codecs.utf_16_le_decode
elif byteorder == 1:
self.decoder = codecs.utf_16_be_decode
elif consumed >= 2:
raise UnicodeError("UTF-16 stream does not start with BOM")
return (output, consumed)
return self.decoder(input, self.errors, final)
Example #12
| Source File: utf_16.py From ironpython3 with Apache License 2.0 | 5 votes |
|
def decode(self, input, errors='strict'):
(object, consumed, byteorder) = \
codecs.utf_16_ex_decode(input, errors, 0, False)
if byteorder == -1:
self.decode = codecs.utf_16_le_decode
elif byteorder == 1:
self.decode = codecs.utf_16_be_decode
elif consumed>=2:
raise UnicodeError("UTF-16 stream does not start with BOM")
return (object, consumed)
### encodings module API
Example #13
| Source File: utf_16_be.py From ironpython3 with Apache License 2.0 | 5 votes |
|
def decode(input, errors='strict'):
return codecs.utf_16_be_decode(input, errors, True)
Example #14
| Source File: test_codecs.py From ironpython3 with Apache License 2.0 | 5 votes |
|
def test_utf_16_be_decode(self):
string, num_processed = codecs.utf_16_be_decode(b'\0a\0b\0c')
self.assertEqual(string, "abc")
self.assertEqual(num_processed, 3 * 2)
string, num_processed = codecs.utf_16_be_decode(codecs.BOM_UTF16_BE + b'\0a\0b\0c')
self.assertEqual(string, "\uFEFFabc")
self.assertEqual(num_processed, 4 * 2)
self.assertEqual(codecs.utf_16_be_decode(array.array('I', (1633771873,))), ("慡慡", 4))
self.assertRaises(TypeError, codecs.utf_16_be_decode, "abc")
self.assertRaises(TypeError, codecs.utf_16_be_decode, None)
self.assertRaises(TypeError, codecs.utf_16_be_decode, None, None)
self.assertEquals(codecs.utf_16_be_decode(b"", None), ("", 0))
Example #15
| Source File: utf_16.py From python with Apache License 2.0 | 5 votes |
|
def getstate(self):
# additional state info from the base class must be None here,
# as it isn't passed along to the caller
state = codecs.BufferedIncrementalDecoder.getstate(self)[0]
# additional state info we pass to the caller:
# 0: stream is in natural order for this platform
# 1: stream is in unnatural order
# 2: endianness hasn't been determined yet
if self.decoder is None:
return (state, 2)
addstate = int((sys.byteorder == "big") !=
(self.decoder is codecs.utf_16_be_decode))
return (state, addstate)
Example #16
| Source File: utf_16.py From python with Apache License 2.0 | 5 votes |
|
def decode(self, input, errors='strict'):
(object, consumed, byteorder) = \
codecs.utf_16_ex_decode(input, errors, 0, False)
if byteorder == -1:
self.decode = codecs.utf_16_le_decode
elif byteorder == 1:
self.decode = codecs.utf_16_be_decode
elif consumed>=2:
raise UnicodeError("UTF-16 stream does not start with BOM")
return (object, consumed)
### encodings module API
Example #17
| Source File: utf_16.py From python with Apache License 2.0 | 5 votes |
|
def setstate(self, state):
# state[1] will be ignored by BufferedIncrementalDecoder.setstate()
codecs.BufferedIncrementalDecoder.setstate(self, state)
state = state[1]
if state == 0:
self.decoder = (codecs.utf_16_be_decode
if sys.byteorder == "big"
else codecs.utf_16_le_decode)
elif state == 1:
self.decoder = (codecs.utf_16_le_decode
if sys.byteorder == "big"
else codecs.utf_16_be_decode)
else:
self.decoder = None
Example #18
| Source File: utf_16.py From Imogen with MIT License | 5 votes |
|
def getstate(self):
# additional state info from the base class must be None here,
# as it isn't passed along to the caller
state = codecs.BufferedIncrementalDecoder.getstate(self)[0]
# additional state info we pass to the caller:
# 0: stream is in natural order for this platform
# 1: stream is in unnatural order
# 2: endianness hasn't been determined yet
if self.decoder is None:
return (state, 2)
addstate = int((sys.byteorder == "big") !=
(self.decoder is codecs.utf_16_be_decode))
return (state, addstate)
Example #19
| Source File: utf_16.py From scylla with Apache License 2.0 | 5 votes |
|
def setstate(self, state):
# state[1] will be ignored by BufferedIncrementalDecoder.setstate()
codecs.BufferedIncrementalDecoder.setstate(self, state)
state = state[1]
if state == 0:
self.decoder = (codecs.utf_16_be_decode
if sys.byteorder == "big"
else codecs.utf_16_le_decode)
elif state == 1:
self.decoder = (codecs.utf_16_le_decode
if sys.byteorder == "big"
else codecs.utf_16_be_decode)
else:
self.decoder = None
Example #20
| Source File: codec.py From naz with MIT License | 5 votes |
|
def decode(input: bytes, errors: str = "strict") -> typing.Tuple[str, int]:
"""
return a string decoded from the given bytes and its length.
Parameters:
input: the bytes to decode
errors: same meaning as the errors argument to pythons' `encode <https://docs.python.org/3/library/codecs.html#codecs.encode>`_ method
"""
return codecs.utf_16_be_decode(input, errors)
Example #21
| Source File: utf_16_be.py From Fluid-Designer with GNU General Public License v3.0 | 5 votes |
|
def decode(input, errors='strict'):
return codecs.utf_16_be_decode(input, errors, True)
Example #22
| Source File: utf_16.py From Fluid-Designer with GNU General Public License v3.0 | 5 votes |
|
def decode(self, input, errors='strict'):
(object, consumed, byteorder) = \
codecs.utf_16_ex_decode(input, errors, 0, False)
if byteorder == -1:
self.decode = codecs.utf_16_le_decode
elif byteorder == 1:
self.decode = codecs.utf_16_be_decode
elif consumed>=2:
raise UnicodeError("UTF-16 stream does not start with BOM")
return (object, consumed)
### encodings module API
Example #23
| Source File: utf_16.py From Fluid-Designer with GNU General Public License v3.0 | 5 votes |
|
def setstate(self, state):
# state[1] will be ignored by BufferedIncrementalDecoder.setstate()
codecs.BufferedIncrementalDecoder.setstate(self, state)
state = state[1]
if state == 0:
self.decoder = (codecs.utf_16_be_decode
if sys.byteorder == "big"
else codecs.utf_16_le_decode)
elif state == 1:
self.decoder = (codecs.utf_16_le_decode
if sys.byteorder == "big"
else codecs.utf_16_be_decode)
else:
self.decoder = None
Example #24
| Source File: utf_16.py From Fluid-Designer with GNU General Public License v3.0 | 5 votes |
|
def getstate(self):
# additonal state info from the base class must be None here,
# as it isn't passed along to the caller
state = codecs.BufferedIncrementalDecoder.getstate(self)[0]
# additional state info we pass to the caller:
# 0: stream is in natural order for this platform
# 1: stream is in unnatural order
# 2: endianness hasn't been determined yet
if self.decoder is None:
return (state, 2)
addstate = int((sys.byteorder == "big") !=
(self.decoder is codecs.utf_16_be_decode))
return (state, addstate)
Example #25
| Source File: utf_16.py From Fluid-Designer with GNU General Public License v3.0 | 5 votes |
|
def _buffer_decode(self, input, errors, final):
if self.decoder is None:
(output, consumed, byteorder) = \
codecs.utf_16_ex_decode(input, errors, 0, final)
if byteorder == -1:
self.decoder = codecs.utf_16_le_decode
elif byteorder == 1:
self.decoder = codecs.utf_16_be_decode
elif consumed >= 2:
raise UnicodeError("UTF-16 stream does not start with BOM")
return (output, consumed)
return self.decoder(input, self.errors, final)
Example #26
| Source File: utf_16_be.py From GraphicDesignPatternByPython with MIT License | 5 votes |
|
def decode(input, errors='strict'):
return codecs.utf_16_be_decode(input, errors, True)
Example #27
| Source File: utf_16.py From GraphicDesignPatternByPython with MIT License | 5 votes |
|
def decode(self, input, errors='strict'):
(object, consumed, byteorder) = \
codecs.utf_16_ex_decode(input, errors, 0, False)
if byteorder == -1:
self.decode = codecs.utf_16_le_decode
elif byteorder == 1:
self.decode = codecs.utf_16_be_decode
elif consumed>=2:
raise UnicodeError("UTF-16 stream does not start with BOM")
return (object, consumed)
### encodings module API
Example #28
| Source File: utf_16.py From GraphicDesignPatternByPython with MIT License | 5 votes |
|
def setstate(self, state):
# state[1] will be ignored by BufferedIncrementalDecoder.setstate()
codecs.BufferedIncrementalDecoder.setstate(self, state)
state = state[1]
if state == 0:
self.decoder = (codecs.utf_16_be_decode
if sys.byteorder == "big"
else codecs.utf_16_le_decode)
elif state == 1:
self.decoder = (codecs.utf_16_le_decode
if sys.byteorder == "big"
else codecs.utf_16_be_decode)
else:
self.decoder = None
Example #29
| Source File: utf_16.py From GraphicDesignPatternByPython with MIT License | 5 votes |
|
def getstate(self):
# additional state info from the base class must be None here,
# as it isn't passed along to the caller
state = codecs.BufferedIncrementalDecoder.getstate(self)[0]
# additional state info we pass to the caller:
# 0: stream is in natural order for this platform
# 1: stream is in unnatural order
# 2: endianness hasn't been determined yet
if self.decoder is None:
return (state, 2)
addstate = int((sys.byteorder == "big") !=
(self.decoder is codecs.utf_16_be_decode))
return (state, addstate)
Example #30
| Source File: utf_16.py From GraphicDesignPatternByPython with MIT License | 5 votes |
|
def _buffer_decode(self, input, errors, final):
if self.decoder is None:
(output, consumed, byteorder) = \
codecs.utf_16_ex_decode(input, errors, 0, final)
if byteorder == -1:
self.decoder = codecs.utf_16_le_decode
elif byteorder == 1:
self.decoder = codecs.utf_16_be_decode
elif consumed >= 2:
raise UnicodeError("UTF-16 stream does not start with BOM")
return (output, consumed)
return self.decoder(input, self.errors, final)
