Python codecs.iterdecode() Examples

def merge_data(self, genes, es, r_server, data_config, es_config):

        #turn the species id/label mappings into a dict from the argument list
        self.orthologs_species = dict()
        if data_config.hgnc_orthologs_species:
            for value in data_config.hgnc_orthologs_species:
                code,label = value.split("-")
                label = label.strip()
                code = code.strip()
                self.orthologs_species[code] = label

        self._logger.info("Ortholog parsing - requesting from URL %s",data_config.hgnc_orthologs)

        with URLZSource(data_config.hgnc_orthologs).open()as source:
            #csv module wants "string" not "unicode" so we have to convert it
            reader = csv.DictReader(codecs.iterdecode(source, 'utf-8'), delimiter="\t")
            for row in reader:
                if row['human_ensembl_gene'] in genes:
                    self.add_ortholog_data_to_gene(gene=genes[row['human_ensembl_gene']], data=row)

        self._logger.info("STATS AFTER HGNC ortholog PARSING:\n" + genes.get_stats()) 

def test_all(self):
        api = (
            "encode", "decode",
            "register", "CodecInfo", "Codec", "IncrementalEncoder",
            "IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
            "getencoder", "getdecoder", "getincrementalencoder",
            "getincrementaldecoder", "getreader", "getwriter",
            "register_error", "lookup_error",
            "strict_errors", "replace_errors", "ignore_errors",
            "xmlcharrefreplace_errors", "backslashreplace_errors",
            "open", "EncodedFile",
            "iterencode", "iterdecode",
            "BOM", "BOM_BE", "BOM_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
            "BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
            "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",  # Undocumented
            "StreamReaderWriter", "StreamRecoder",
        )
        self.assertCountEqual(api, codecs.__all__)
        for api in codecs.__all__:
            getattr(codecs, api) 

def test_all(self):
        api = (
            "encode", "decode",
            "register", "CodecInfo", "Codec", "IncrementalEncoder",
            "IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
            "getencoder", "getdecoder", "getincrementalencoder",
            "getincrementaldecoder", "getreader", "getwriter",
            "register_error", "lookup_error",
            "strict_errors", "replace_errors", "ignore_errors",
            "xmlcharrefreplace_errors", "backslashreplace_errors",
            "namereplace_errors",
            "open", "EncodedFile",
            "iterencode", "iterdecode",
            "BOM", "BOM_BE", "BOM_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
            "BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
            "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",  # Undocumented
            "StreamReaderWriter", "StreamRecoder",
        )
        self.assertCountEqual(api, codecs.__all__)
        for api in codecs.__all__:
            getattr(codecs, api) 

def ticker_universe(self) -> List[str]:
        """fetch universe of accepted tickers

        Returns:
            list of strings: list of currently accepted tickers

        Example:
            >>> SignalsAPI().ticker_universe()
            ["MSFT", "AMZN", "APPL", ...]
        """
        domain = 'https://numerai-quant-public-data.s3-us-west-2.amazonaws.com'
        url = f"{domain}/example_predictions/latest.csv"
        result = requests.get(url, stream=True)
        iterator = codecs.iterdecode(result.iter_lines(), 'utf-8')
        reader = csv.reader(iterator, delimiter=',', quotechar='"')
        tickers = [t for t, _ in reader]
        return tickers 

def test_all(self):
        api = (
            "encode", "decode",
            "register", "CodecInfo", "Codec", "IncrementalEncoder",
            "IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
            "getencoder", "getdecoder", "getincrementalencoder",
            "getincrementaldecoder", "getreader", "getwriter",
            "register_error", "lookup_error",
            "strict_errors", "replace_errors", "ignore_errors",
            "xmlcharrefreplace_errors", "backslashreplace_errors",
            "namereplace_errors",
            "open", "EncodedFile",
            "iterencode", "iterdecode",
            "BOM", "BOM_BE", "BOM_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
            "BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
            "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",  # Undocumented
            "StreamReaderWriter", "StreamRecoder",
        )
        self.assertCountEqual(api, codecs.__all__)
        for api in codecs.__all__:
            getattr(codecs, api) 

def query_csv(self, query: str, org=None, dialect: Dialect = default_dialect):
        """
        Executes the Flux query and return results as a CSV iterator. Each iteration returns a row of the CSV file.

        :param query: a Flux query
        :param org: organization name (optional if already specified in InfluxDBClient)
        :param dialect: csv dialect format
        :return: The returned object is an iterator.  Each iteration returns a row of the CSV file
                 (which can span multiple input lines).
        """
        if org is None:
            org = self._influxdb_client.org
        response = self._query_api.post_query(org=org, query=self._create_query(query, dialect), async_req=False,
                                              _preload_content=False)

        return csv.reader(codecs.iterdecode(response, 'utf-8')) 

def encode_all(f=None, **kwargs):
    """
    Encode unicode into bytes (str)
    """
    names = kwargs.pop('fieldnames', None)
    encoding = kwargs.pop('encoding', None) if f else False

    if PY2:
        decoded = codecs.iterdecode(f, encoding) if encoding else f
        ekwargs = {encode(k): encode(v) for k, v in kwargs.items()}
    else:
        decoded, ekwargs = f, kwargs

    res = {
        'f': codecs.iterencode(decoded, ENCODING) if f and PY2 else decoded,
        'fieldnames': [encode(x) for x in names] if names and PY2 else names,
        'drkwargs': use_keys_from(ekwargs, READER_KEYS),
        'dwkwargs': use_keys_from(ekwargs, WRITER_KEYS),
        'fmtparams': use_keys_from(ekwargs, FMTKEYS)}

    return res 

def test_all(self):
        api = (
            "encode", "decode",
            "register", "CodecInfo", "Codec", "IncrementalEncoder",
            "IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
            "getencoder", "getdecoder", "getincrementalencoder",
            "getincrementaldecoder", "getreader", "getwriter",
            "register_error", "lookup_error",
            "strict_errors", "replace_errors", "ignore_errors",
            "xmlcharrefreplace_errors", "backslashreplace_errors",
            "open", "EncodedFile",
            "iterencode", "iterdecode",
            "BOM", "BOM_BE", "BOM_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
            "BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
            "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",  # Undocumented
            "StreamReaderWriter", "StreamRecoder",
        )
        self.assertEqual(sorted(api), sorted(codecs.__all__))
        for api in codecs.__all__:
            getattr(codecs, api) 

def test_refresh():
    """
    Test token expiration and refresh.
    """
    test_client = make_test_client()

    with patch('time.time', Mock(return_value=time.time())) as time_1:
        # authenticate and get an ID token cookie
        auth_redirect = test_client.get('/')
        callback_redirect = test_client.get(callback_url_for(auth_redirect))
        actual_page = test_client.get(callback_redirect.headers['Location'])
        page_text = ''.join(codecs.iterdecode(actual_page.response, 'utf-8'))
        assert page_text == 'too many secrets', "Authentication failed"

    # app should now try to use the refresh token
    with patch('time.time', Mock(return_value=time.time() + 10)) as time_2:
        test_client.get('/')
        body = parse_qs(last_request['body'])
        assert body.get('refresh_token') == ['mock_refresh_token'],\
            "App should have tried to refresh credentials" 

def reopen(f, encoding):
    sanitized_encoding = sanitize_file_encoding(encoding)
    logger.debug('Reopening %s with encoding: %s', f, sanitized_encoding)

    try:
        decoded_f = open(f.name, encoding=sanitized_encoding)
    except AttributeError:
        f.seek(0)
        decoded_f = iterdecode(f, sanitized_encoding)

    return decoded_f 

def test_incremental_decode(self):
        self.assertEqual(
            "".join(codecs.iterdecode("python.org", "idna")),
            u"python.org"
        )
        self.assertEqual(
            "".join(codecs.iterdecode("python.org.", "idna")),
            u"python.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )

        decoder = codecs.getincrementaldecoder("idna")()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode(u"rg"), u"")
        self.assertEqual(decoder.decode(u"", True), u"org")

        decoder.reset()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode("rg."), u"org.")
        self.assertEqual(decoder.decode("", True), u"") 

def test_incremental_decode(self):
        self.assertEqual(
            "".join(codecs.iterdecode("python.org", "idna")),
            u"python.org"
        )
        self.assertEqual(
            "".join(codecs.iterdecode("python.org.", "idna")),
            u"python.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )

        decoder = codecs.getincrementaldecoder("idna")()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode(u"rg"), u"")
        self.assertEqual(decoder.decode(u"", True), u"org")

        decoder.reset()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode("rg."), u"org.")
        self.assertEqual(decoder.decode("", True), u"") 

def __enter__(self):
        self._reader = csv_parser.reader(codecs.iterdecode(self._response, 'utf-8'))
        return self 

def _fetch(self):
        feed = {}
        try:
            for base in self.bases:
                feed[base] = {}

                for quote in self.quotes:
                    if quote == base:
                        continue

                    ticker = "%s%s" % (quote, base)
                    url = (
                        'http://www.google.com/finance/getprices'
                        '?i={period}&p={days}d&f=d,c&df=cpct&q={ticker}'
                    ).format(ticker=ticker, period=self.period, days=self.days)

                    response = requests.get(url=url, headers=_request_headers, timeout=self.timeout)
                    reader = csv.reader(codecs.iterdecode(response.content.splitlines(), "utf-8"))

                    prices = []
                    for row in reader:
                        if re.match('^[a\d]', row[0]):
                            prices.append(float(row[1]))

                    if hasattr(self, "quoteNames") and quote in self.quoteNames:
                        quote = self.quoteNames[quote]
                    feed[base][quote] = {"price": sum(prices) / len(prices),
                                         "volume": 1.0}
        except Exception as e:
            raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
        return feed 

def wait_and_save_report(self, report):
        if self.args.save:
            output = open(self.args.save, "wb")
            print("Opened %s for writing, requesting report")
            with closing(output):
                report.wait()
                report.download().write_to(output)
        else:
            report.wait()
            download = report.download()
            with closing(download):
                lines = codecs.iterdecode(iter(download), 'UTF-8')
                for line in lines:
                    print(line) 

def test_incremental_decode(self):
        self.assertEqual(
            "".join(codecs.iterdecode((bytes([c]) for c in b"python.org"), "idna")),
            "python.org"
        )
        self.assertEqual(
            "".join(codecs.iterdecode((bytes([c]) for c in b"python.org."), "idna")),
            "python.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode((bytes([c]) for c in b"xn--pythn-mua.org."), "idna")),
            "pyth\xf6n.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode((bytes([c]) for c in b"xn--pythn-mua.org."), "idna")),
            "pyth\xf6n.org."
        )

        decoder = codecs.getincrementaldecoder("idna")()
        self.assertEqual(decoder.decode(b"xn--xam", ), "")
        self.assertEqual(decoder.decode(b"ple-9ta.o", ), "\xe4xample.")
        self.assertEqual(decoder.decode(b"rg"), "")
        self.assertEqual(decoder.decode(b"", True), "org")

        decoder.reset()
        self.assertEqual(decoder.decode(b"xn--xam", ), "")
        self.assertEqual(decoder.decode(b"ple-9ta.o", ), "\xe4xample.")
        self.assertEqual(decoder.decode(b"rg."), "org.")
        self.assertEqual(decoder.decode(b"", True), "") 

def test_basics_capi(self):
        from _testcapi import codec_incrementalencoder, codec_incrementaldecoder
        s = u"abc123"  # all codecs should be able to encode these
        for encoding in all_unicode_encodings:
            if encoding not in broken_incremental_coders:
                # check incremental decoder/encoder and iterencode()/iterdecode()
                try:
                    cencoder = codec_incrementalencoder(encoding)
                except LookupError:  # no IncrementalEncoder
                    pass
                else:
                    # check C API
                    encodedresult = ""
                    for c in s:
                        encodedresult += cencoder.encode(c)
                    encodedresult += cencoder.encode(u"", True)
                    cdecoder = codec_incrementaldecoder(encoding)
                    decodedresult = u""
                    for c in encodedresult:
                        decodedresult += cdecoder.decode(c)
                    decodedresult += cdecoder.decode("", True)
                    self.assertEqual(decodedresult, s,
                                     "encoding=%r" % encoding)

                if encoding not in only_strict_mode:
                    # check incremental decoder/encoder with errors argument
                    try:
                        cencoder = codec_incrementalencoder(encoding, "ignore")
                    except LookupError:  # no IncrementalEncoder
                        pass
                    else:
                        encodedresult = "".join(cencoder.encode(c) for c in s)
                        cdecoder = codec_incrementaldecoder(encoding, "ignore")
                        decodedresult = u"".join(cdecoder.decode(c)
                                                 for c in encodedresult)
                        self.assertEqual(decodedresult, s,
                                         "encoding=%r" % encoding) 

def test_incremental_decode(self):
        self.assertEqual(
            "".join(codecs.iterdecode("python.org", "idna")),
            u"python.org"
        )
        self.assertEqual(
            "".join(codecs.iterdecode("python.org.", "idna")),
            u"python.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )

        decoder = codecs.getincrementaldecoder("idna")()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode(u"rg"), u"")
        self.assertEqual(decoder.decode(u"", True), u"org")

        decoder.reset()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode("rg."), u"org.")
        self.assertEqual(decoder.decode("", True), u"") 

def read_file_as_csv(file):
    reader = csv.reader(codecs.iterdecode(file, 'utf-8'))
    return reader 

def write_encoded_data(self, f):
    # Sometimes users give us invalid utf-8 data. They shouldn't, but it does
    # happen every once and a while. Just ignore it, and replace with �.
    # We're assuming users only want to write text data out.
    # self.data can be large, so be careful to do the conversion in chunks
    # while streaming the data out, instead of requiring a full copy.
    n = 1 << 16
    # This is a generator expression, so this only copies one chunk of
    # self.data at any one time.
    chunks = (self.data[i:i + n] for i in xrange(0, len(self.data), n))
    decoded = codecs.iterdecode(chunks, 'utf-8', 'replace')
    for chunk in codecs.iterencode(decoded, 'utf-8'):
      f.write(chunk) 

def test_incremental_decode(self):
        self.assertEqual(
            "".join(codecs.iterdecode("python.org", "idna")),
            u"python.org"
        )
        self.assertEqual(
            "".join(codecs.iterdecode("python.org.", "idna")),
            u"python.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )

        decoder = codecs.getincrementaldecoder("idna")()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode(u"rg"), u"")
        self.assertEqual(decoder.decode(u"", True), u"org")

        decoder.reset()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode("rg."), u"org.")
        self.assertEqual(decoder.decode("", True), u"") 

def read_decoded_data(self, f):
    # This ensures that the raw result bytes we got are, in fact, valid utf-8,
    # replacing invalid bytes with �. Because python2's unicode support is
    # wonky, we re-encode the now-valid-utf-8 back into a str object so that
    # users don't need to deal with `unicode` objects.
    # The file contents can be large, so be careful to do the conversion in
    # chunks while streaming the data in, instead of requiring a full copy.
    n = 1 << 16
    chunks = iter(lambda: f.read(n), '')
    decoded = codecs.iterdecode(chunks, 'utf-8', 'replace')
    return ''.join(codecs.iterencode(decoded, 'utf-8')) 

def test_signin():
    """
    Happy path authentication test.
    """
    test_client = make_test_client()

    # make an unauthenticated request,
    # which should result in a redirect to the IdP
    r1 = test_client.get('/')
    assert r1.status_code == 302,\
        "Expected redirect to IdP "\
        "(response status was {response.status})".format(response=r1)

    # the app should now contact the IdP
    # to exchange that auth code for credentials
    r2 = test_client.get(callback_url_for(r1))
    assert r2.status_code == 302,\
        "Expected redirect to destination "\
        "(response status was {response.status})".format(response=r2)
    r2location = urlsplit(r2.headers['Location'])
    assert r2location.path == '/',\
        "Expected redirect to destination "\
        "(unexpected path {location.path})".format(location=r2location)

    # Let's get the at and rt
    r3 = test_client.get('/at')
    assert r3.status_code == 200,\
        "Expected access token to succeed"
    page_text = ''.join(codecs.iterdecode(r3.response, 'utf-8'))
    assert page_text == 'mock_access_token',\
        "Access token expected"
    r4 = test_client.get('/rt')
    assert r4.status_code == 200,\
        "Expected refresh token to succeed"
    page_text = ''.join(codecs.iterdecode(r4.response, 'utf-8'))
    assert page_text == 'mock_refresh_token',\
        "Refresh token expected" 

def test_incremental_decode(self):
        self.assertEqual(
            "".join(codecs.iterdecode("python.org", "idna")),
            u"python.org"
        )
        self.assertEqual(
            "".join(codecs.iterdecode("python.org.", "idna")),
            u"python.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )

        decoder = codecs.getincrementaldecoder("idna")()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode(u"rg"), u"")
        self.assertEqual(decoder.decode(u"", True), u"org")

        decoder.reset()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode("rg."), u"org.")
        self.assertEqual(decoder.decode("", True), u"") 

def test_basics_capi(self):
        from _testcapi import codec_incrementalencoder, codec_incrementaldecoder
        s = u"abc123"  # all codecs should be able to encode these
        for encoding in all_unicode_encodings:
            if encoding not in broken_incremental_coders:
                # check incremental decoder/encoder and iterencode()/iterdecode()
                try:
                    cencoder = codec_incrementalencoder(encoding)
                except LookupError:  # no IncrementalEncoder
                    pass
                else:
                    # check C API
                    encodedresult = ""
                    for c in s:
                        encodedresult += cencoder.encode(c)
                    encodedresult += cencoder.encode(u"", True)
                    cdecoder = codec_incrementaldecoder(encoding)
                    decodedresult = u""
                    for c in encodedresult:
                        decodedresult += cdecoder.decode(c)
                    decodedresult += cdecoder.decode("", True)
                    self.assertEqual(decodedresult, s,
                                     "encoding=%r" % encoding)

                if encoding not in only_strict_mode:
                    # check incremental decoder/encoder with errors argument
                    try:
                        cencoder = codec_incrementalencoder(encoding, "ignore")
                    except LookupError:  # no IncrementalEncoder
                        pass
                    else:
                        encodedresult = "".join(cencoder.encode(c) for c in s)
                        cdecoder = codec_incrementaldecoder(encoding, "ignore")
                        decodedresult = u"".join(cdecoder.decode(c)
                                                 for c in encodedresult)
                        self.assertEqual(decodedresult, s,
                                         "encoding=%r" % encoding) 

def _fetch_event_source_data(self, url):
        # stream=True allows lazy iteration
        response = requests.get(url, stream=True)
        response_iter = response.iter_lines()
        # CSV reader wants str instead of byte, let's decode
        decoded_response_iter = codecs.iterdecode(response_iter, 'utf-8')
        reader = csv.DictReader(decoded_response_iter, delimiter=';', quotechar='"', doublequote=True)
        return reader 

def test_incremental_decode(self):
        self.assertEqual(
            "".join(codecs.iterdecode((bytes([c]) for c in b"python.org"), "idna")),
            "python.org"
        )
        self.assertEqual(
            "".join(codecs.iterdecode((bytes([c]) for c in b"python.org."), "idna")),
            "python.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode((bytes([c]) for c in b"xn--pythn-mua.org."), "idna")),
            "pyth\xf6n.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode((bytes([c]) for c in b"xn--pythn-mua.org."), "idna")),
            "pyth\xf6n.org."
        )

        decoder = codecs.getincrementaldecoder("idna")()
        self.assertEqual(decoder.decode(b"xn--xam", ), "")
        self.assertEqual(decoder.decode(b"ple-9ta.o", ), "\xe4xample.")
        self.assertEqual(decoder.decode(b"rg"), "")
        self.assertEqual(decoder.decode(b"", True), "org")

        decoder.reset()
        self.assertEqual(decoder.decode(b"xn--xam", ), "")
        self.assertEqual(decoder.decode(b"ple-9ta.o", ), "\xe4xample.")
        self.assertEqual(decoder.decode(b"rg."), "org.")
        self.assertEqual(decoder.decode(b"", True), "") 

def test_incremental_decode(self):
        self.assertEqual(
            "".join(codecs.iterdecode("python.org", "idna")),
            u"python.org"
        )
        self.assertEqual(
            "".join(codecs.iterdecode("python.org.", "idna")),
            u"python.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )
        self.assertEqual(
            "".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
            u"pyth\xf6n.org."
        )

        decoder = codecs.getincrementaldecoder("idna")()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode(u"rg"), u"")
        self.assertEqual(decoder.decode(u"", True), u"org")

        decoder.reset()
        self.assertEqual(decoder.decode("xn--xam", ), u"")
        self.assertEqual(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
        self.assertEqual(decoder.decode("rg."), u"org.")
        self.assertEqual(decoder.decode("", True), u"") 

def test_basics_capi(self):
        from _testcapi import codec_incrementalencoder, codec_incrementaldecoder
        s = u"abc123"  # all codecs should be able to encode these
        for encoding in all_unicode_encodings:
            if encoding not in broken_incremental_coders:
                # check incremental decoder/encoder and iterencode()/iterdecode()
                try:
                    cencoder = codec_incrementalencoder(encoding)
                except LookupError:  # no IncrementalEncoder
                    pass
                else:
                    # check C API
                    encodedresult = ""
                    for c in s:
                        encodedresult += cencoder.encode(c)
                    encodedresult += cencoder.encode(u"", True)
                    cdecoder = codec_incrementaldecoder(encoding)
                    decodedresult = u""
                    for c in encodedresult:
                        decodedresult += cdecoder.decode(c)
                    decodedresult += cdecoder.decode("", True)
                    self.assertEqual(decodedresult, s,
                                     "encoding=%r" % encoding)

                if encoding not in only_strict_mode:
                    # check incremental decoder/encoder with errors argument
                    try:
                        cencoder = codec_incrementalencoder(encoding, "ignore")
                    except LookupError:  # no IncrementalEncoder
                        pass
                    else:
                        encodedresult = "".join(cencoder.encode(c) for c in s)
                        cdecoder = codec_incrementaldecoder(encoding, "ignore")
                        decodedresult = u"".join(cdecoder.decode(c)
                                                 for c in encodedresult)
                        self.assertEqual(decodedresult, s,
                                         "encoding=%r" % encoding) 

def _prim_fromgen(self, yielder, info):
        chars = []
        # not exactly a model of efficiency...
        for c in codecs.iterdecode(yielder, self._vs_encoding):
            if ord(c) == 0:
                break
            chars.append(c)

        self.vsResize(info['size'])
        return ''.join(chars)