Python codecs.StreamRecoder() Examples

The following are 6 code examples of codecs.StreamRecoder(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module codecs , or try the search function .
Example #1
Source File: test_common.py    From recruit with Apache License 2.0 6 votes vote down vote up 
def test_bad_stream_exception(all_parsers, csv_dir_path):
    # see gh-13652
    #
    # This test validates that both the Python engine and C engine will
    # raise UnicodeDecodeError instead of C engine raising ParserError
    # and swallowing the exception that caused read to fail.
    path = os.path.join(csv_dir_path, "sauron.SHIFT_JIS.csv")
    codec = codecs.lookup("utf-8")
    utf8 = codecs.lookup('utf-8')
    parser = all_parsers

    msg = ("'utf-8' codec can't decode byte" if compat.PY3
           else "'utf8' codec can't decode byte")

    # Stream must be binary UTF8.
    with open(path, "rb") as handle, codecs.StreamRecoder(
            handle, utf8.encode, utf8.decode, codec.streamreader,
            codec.streamwriter) as stream:

        with pytest.raises(UnicodeDecodeError, match=msg):
            parser.read_csv(stream)
Example #2
Source File: common.py    From vnpy_crypto with MIT License 6 votes vote down vote up 
def test_bad_stream_exception(self):
        # Issue 13652:
        # This test validates that both python engine
        # and C engine will raise UnicodeDecodeError instead of
        # c engine raising ParserError and swallowing exception
        # that caused read to fail.
        codec = codecs.lookup("utf-8")
        utf8 = codecs.lookup('utf-8')

        if compat.PY3:
            msg = "'utf-8' codec can't decode byte"
        else:
            msg = "'utf8' codec can't decode byte"

        # stream must be binary UTF8
        with open(self.csv_shiftjs, "rb") as handle, codecs.StreamRecoder(
                handle, utf8.encode, utf8.decode, codec.streamreader,
                codec.streamwriter) as stream:

            with tm.assert_raises_regex(UnicodeDecodeError, msg):
                self.read_csv(stream)
Example #3
Source File: test_common.py    From predictive-maintenance-using-machine-learning with Apache License 2.0 6 votes vote down vote up 
def test_bad_stream_exception(all_parsers, csv_dir_path):
    # see gh-13652
    #
    # This test validates that both the Python engine and C engine will
    # raise UnicodeDecodeError instead of C engine raising ParserError
    # and swallowing the exception that caused read to fail.
    path = os.path.join(csv_dir_path, "sauron.SHIFT_JIS.csv")
    codec = codecs.lookup("utf-8")
    utf8 = codecs.lookup('utf-8')
    parser = all_parsers

    msg = ("'utf-8' codec can't decode byte" if compat.PY3
           else "'utf8' codec can't decode byte")

    # Stream must be binary UTF8.
    with open(path, "rb") as handle, codecs.StreamRecoder(
            handle, utf8.encode, utf8.decode, codec.streamreader,
            codec.streamwriter) as stream:

        with pytest.raises(UnicodeDecodeError, match=msg):
            parser.read_csv(stream)
Example #4
Source File: common.py    From elasticintel with GNU General Public License v3.0 6 votes vote down vote up 
def test_bad_stream_exception(self):
        # Issue 13652:
        # This test validates that both python engine
        # and C engine will raise UnicodeDecodeError instead of
        # c engine raising ParserError and swallowing exception
        # that caused read to fail.
        handle = open(self.csv_shiftjs, "rb")
        codec = codecs.lookup("utf-8")
        utf8 = codecs.lookup('utf-8')
        # stream must be binary UTF8
        stream = codecs.StreamRecoder(
            handle, utf8.encode, utf8.decode, codec.streamreader,
            codec.streamwriter)
        if compat.PY3:
            msg = "'utf-8' codec can't decode byte"
        else:
            msg = "'utf8' codec can't decode byte"
        with tm.assert_raises_regex(UnicodeDecodeError, msg):
            self.read_csv(stream)
        stream.close()
Example #5
Source File: common.py    From twitter-stock-recommendation with MIT License 6 votes vote down vote up 
def test_bad_stream_exception(self):
        # Issue 13652:
        # This test validates that both python engine
        # and C engine will raise UnicodeDecodeError instead of
        # c engine raising ParserError and swallowing exception
        # that caused read to fail.
        codec = codecs.lookup("utf-8")
        utf8 = codecs.lookup('utf-8')

        if compat.PY3:
            msg = "'utf-8' codec can't decode byte"
        else:
            msg = "'utf8' codec can't decode byte"

        # stream must be binary UTF8
        with open(self.csv_shiftjs, "rb") as handle, codecs.StreamRecoder(
                handle, utf8.encode, utf8.decode, codec.streamreader,
                codec.streamwriter) as stream:

            with tm.assert_raises_regex(UnicodeDecodeError, msg):
                self.read_csv(stream)
Example #6
Source File: reader.py    From batch-scoring with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
def __init__(self, f, encoding):
        f.seek(0)
        if six.PY3:
            self.reader = f
        if six.PY2:
            self.reader = codecs.StreamRecoder(f,
                                               codecs.getencoder('utf-8'),
                                               codecs.getdecoder('utf-8'),
                                               codecs.getreader(encoding),
                                               codecs.getwriter(encoding))