Python requests.exceptions.ChunkedEncodingError() Examples

def listen_for_dweets_from(thing_name, timeout=900, key=None, session=None):
    """Create a real-time subscription to dweets
    """
    url = BASE_URL + '/listen/for/dweets/from/{0}'.format(thing_name)
    session = session or requests.Session()
    if key is not None:
        params = {'key': key}
    else:
        params = None

    start = datetime.datetime.utcnow()
    while True:
        request = requests.Request("GET", url, params=params).prepare()
        resp = session.send(request, stream=True, timeout=timeout)
        try:
            for x in _listen_for_dweets_from_response(resp):
                yield x
                _check_stream_timeout(start, timeout)
        except (ChunkedEncodingError, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
            pass
        _check_stream_timeout(start, timeout) 

def test_download_corrupted_obb(self, playstore, download_folder_path, monkeypatch):
        original = Util.show_list_progress

        def raise_exception(*args, **kwargs):
            if " .obb ".lower() not in kwargs["description"].lower():
                return original(*args, **kwargs)
            else:
                raise ChunkedEncodingError()

        monkeypatch.setattr(Util, "show_list_progress", raise_exception)

        result = playstore.download(
            APK_WITH_OBB,
            os.path.join(download_folder_path, f"{APK_WITH_OBB}.apk"),
            download_obb=True,
            show_progress_bar=False,
        )
        assert result is False 

def post_stream(url, data=None, headers=None, encoding='UTF-8', files=None):
    """分块接受数据"""
    try:
        lines = requests.post(url, data=data, headers=headers, timeout=_tiemout, stream=True, proxies=proxies,
                              files=None)
        html = list()
        for line in lines.iter_lines():
            line = line.decode(encoding)
            html.append(line.strip())
        return '\r\n'.join(html).strip()
    except ChunkedEncodingError as e:
        return '\r\n'.join(html).strip()
    except ConnectionError as e:
        return "ERROR:" + "HTTP连接错误"
    except ConnectTimeout as e:
        return "ERROR:" + "HTTP连接超时错误"
    except Exception as e:
        return 'ERROR:' + str(e) 

def test_download_corrupted_split_apk(
        self, playstore, download_folder_path, monkeypatch
    ):
        original = Util.show_list_progress

        def raise_exception(*args, **kwargs):
            if " split apk ".lower() not in kwargs["description"].lower():
                return original(*args, **kwargs)
            else:
                raise ChunkedEncodingError()

        monkeypatch.setattr(Util, "show_list_progress", raise_exception)

        result = playstore.download(
            APK_WITH_SPLIT_APK,
            os.path.join(download_folder_path, f"{APK_WITH_SPLIT_APK}.apk"),
            download_split_apks=True,
            show_progress_bar=False,
        )
        assert result is False 

def get_stream(url, headers=None, encoding='UTF-8'):
    """分块接受数据"""
    try:
        lines = requests.get(url, headers=headers, timeout=_tiemout, stream=True, proxies=proxies)
        html = list()
        for line in lines.iter_lines():
            if b'\x00' in line:
                break
            line = line.decode(encoding)
            html.append(line.strip())
        return '\r\n'.join(html).strip()
    except ChunkedEncodingError as e:
        return '\r\n'.join(html).strip()
    except ConnectionError as e:
        return "ERROR:" + "HTTP连接错误"
    except ConnectTimeout as e:
        return "ERROR:" + "HTTP连接超时错误"
    except Exception as e:
        return 'ERROR:' + str(e) 

def test_download_corrupted_apk(self, playstore, download_folder_path, monkeypatch):
        # noinspection PyUnusedLocal
        def raise_exception(*args, **kwargs):
            raise ChunkedEncodingError()

        monkeypatch.setattr(Util, "show_list_progress", raise_exception)

        # Mock the function that gets the size of the file so that the downloaded
        # apk will be treated as corrupted.
        monkeypatch.setattr(os.path, "getsize", lambda x: 1)

        # Simulate an error with the file deletion.
        # noinspection PyUnusedLocal
        def raise_os_error(ignore):
            raise OSError

        monkeypatch.setattr(os, "remove", raise_os_error)

        result = playstore.download(
            VALID_PACKAGE_NAME,
            os.path.join(download_folder_path, f"{VALID_PACKAGE_NAME}.apk"),
        )
        assert result is False 

def post_stream(url, data=None, headers=None, encoding='UTF-8', files=None):
    """分块接受数据"""
    try:
        lines = requests.post(url, data=data, headers=headers, timeout=_tiemout, stream=True, proxies=proxies,
                              files=None)
        html = list()
        for line in lines.iter_lines():
            line = line.decode(encoding)
            html.append(line.strip())
        return '\r\n'.join(html).strip()
    except ChunkedEncodingError as e:
        return '\r\n'.join(html).strip()
    except ConnectionError as e:
        return "ERROR:" + "HTTP连接错误"
    except ConnectTimeout as e:
        return "ERROR:" + "HTTP连接超时错误"
    except Exception as e:
        return 'ERROR:' + str(e) 

def connection_handler(session, request, verify, as_is_reply=False):
    air = as_is_reply
    s = session
    r = request
    v = verify
    return_json = False

    disable_warnings(InsecureRequestWarning)

    try:
        get = s.send(r, verify=v)
        if get.status_code == 401:
            if 'NoSiteContext' in str(get.content):
                logger.info('Your Site is incorrect for %s', r.url)
            elif 'LoginRequired' in str(get.content):
                logger.info('Your login credentials are incorrect for %s', r.url)
            else:
                logger.info('Your api key is incorrect for %s', r.url)
        elif get.status_code == 404:
            logger.info('This url doesnt even resolve: %s', r.url)
        elif get.status_code == 200:
            try:
                return_json = get.json()
            except JSONDecodeError:
                logger.error('No JSON response. Response is: %s', get.text)
        if air:
            return get
    except InvalidSchema:
        logger.error("You added http(s):// in the config file. Don't do that.")
    except SSLError as e:
        logger.error('Either your host is unreachable or you have an SSL issue. : %s', e)
    except ConnectionError as e:
        logger.error('Cannot resolve the url/ip/port. Check connectivity. Error: %s', e)
    except ChunkedEncodingError as e:
        logger.error('Broken connection during request... oops? Error: %s', e)

    return return_json 

def __retry(self, request, retries=0):
        try:
            return request()
        except ChunkedEncodingError as e:
            if retries != self.max_retries:
                return self.__retry(request, retries=retries + 1)
            else:
                raise e 

def _get_member_by_mid(self, mid: int) -> Optional[dict]:
        """
        根据用户id获取其信息
        :param mid: B站用户id
        :return: 用户详情 or None
        """
        get_params = {
            'mid': mid,
            'jsonp': 'jsonp'
        }
        try:
            res_json = requests.get(API_MEMBER_INFO, params=get_params, timeout=WAIT_MAX, proxies=self.cur_proxy,
                                    headers=self.headers).json()
        except ConnectTimeout as e:
            print(f'获取用户id: {mid} 详情失败: 请求接口超时, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ReadTimeout as e:
            print(f'获取用户id: {mid} 详情失败: 接口读取超时, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ValueError as e:
            # 解析json失败基本上就是ip被封了
            print(f'获取用户id: {mid} 详情失败: 解析json出错, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ProxyError as e:
            print(f'获取用户id: {mid} 详情失败: 连接代理失败, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except requests.ConnectionError as e:
            # 可以断定就是代理IP地址无效
            print(f'获取用户id: {mid} 详情失败: 连接错误, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ChunkedEncodingError as e:
            print(f'获取用户id: {mid} 详情失败: 远程主机强迫关闭了一个现有的连接, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        else:
            if res_json['code'] == -404:
                print(f'找不到用户mid:{mid}')
                raise UserNotFoundException(f'找不到用户mid:{mid}')
            if 'data' in res_json:
                return res_json['data']
            print(f'获取用户id: {mid} 详情失败: data字段不存在!')
        return 

def test_watch_client_request_failed(self, m_sys_exit):
        path = '/test'
        m_handler = mock.Mock()
        watcher_obj = self._test_watch_create_watcher(path, m_handler)
        watcher_obj._watch(path)
        self.client.watch.side_effect = exceptions.ChunkedEncodingError(
            "Connection Broken")

        self.client.watch.assert_called_once()
        self.assertFalse(watcher_obj._alive)
        m_sys_exit.assert_called_once_with(1) 

def write(self, sequence, res, chunk_size=8192):
        if sequence.segment.key and sequence.segment.key.method != "NONE":
            try:
                decryptor = self.create_decryptor(sequence.segment.key,
                                                  sequence.num)
            except StreamError as err:
                log.error("Failed to create decryptor: {0}", err)
                self.close()
                return

            data = res.content
            # If the input data is not a multiple of 16, cut off any garbage
            garbage_len = len(data) % 16
            if garbage_len:
                log.debug("Cutting off {0} bytes of garbage "
                          "before decrypting", garbage_len)
                decrypted_chunk = decryptor.decrypt(data[:-garbage_len])
            else:
                decrypted_chunk = decryptor.decrypt(data)

            self.reader.buffer.write(pkcs7_decode(decrypted_chunk))
        else:
            try:
                for chunk in res.iter_content(chunk_size):
                    self.reader.buffer.write(chunk)
            except ChunkedEncodingError:
                log.error("Download of segment {0} failed", sequence.num)

                return

        log.debug("Download of segment {0} complete", sequence.num) 

def post(url, data=None, headers=None, encoding='UTF-8', files=None):
    """POST请求发送包装"""
    try:
        html = requests.post(url, data=data, headers=headers, proxies=proxies, timeout=_tiemout, files=files)
        html = html.content.decode(encoding)
        return html.replace('\x00', '').strip()
    except ChunkedEncodingError as e:
        html = post_stream(url, data, headers, encoding, files)
        return html
    except ConnectionError as e:
        return "ERROR:" + "HTTP连接错误"
    except ConnectTimeout as e:
        return "ERROR:" + "HTTP连接超时错误"
    except Exception as e:
        return 'ERROR:' + str(e) 

def get(url, headers=None, encoding='UTF-8'):
    """GET请求发送包装"""
    try:
        html = requests.get(url, headers=headers, proxies=proxies, timeout=_tiemout)
        html = html.content.decode(encoding)
        return html.replace('\x00', '').strip()
    except ChunkedEncodingError as e:
        html = get_stream(url, headers, encoding)
        return html
    except ConnectionError as e:
        return "ERROR:" + "HTTP连接错误"
    except ConnectTimeout as e:
        return "ERROR:" + "HTTP连接超时错误"
    except Exception as e:
        return 'ERROR:' + str(e) 

def post(url, data=None, headers=None, encoding='UTF-8', files=None):
    """POST请求发送包装"""
    try:
        html = requests.post(url, data=data, headers=headers, proxies=proxies, timeout=_tiemout, files=files)
        html = html.content.decode(encoding)
        return html.replace('\x00', '').strip()
    except ChunkedEncodingError as e:
        html = post_stream(url, data, headers, encoding, files)
        return html
    except ConnectionError as e:
        return "ERROR:" + "HTTP连接错误"
    except ConnectTimeout as e:
        return "ERROR:" + "HTTP连接超时错误"
    except Exception as e:
        return 'ERROR:' + str(e) 

def get_stream(url, headers=None, encoding='UTF-8'):
    """分块接受数据"""
    try:
        lines = requests.get(url, headers=headers, timeout=_tiemout, stream=True, proxies=proxies)
        html = list()
        for line in lines.iter_lines():
            if b'\x00' in line:
                break
            line = line.decode(encoding)
            html.append(line.strip())
        return '\r\n'.join(html).strip()
    except ChunkedEncodingError as e:
        return '\r\n'.join(html).strip()
    except ConnectionError as e:
        return "ERROR:" + "HTTP连接错误"
    except ConnectTimeout as e:
        return "ERROR:" + "HTTP连接超时错误"
    except Exception as e:
        return 'ERROR:' + str(e) 

def get(url, headers=None, encoding='UTF-8'):
    """GET请求发送包装"""
    try:
        html = requests.get(url, headers=headers, proxies=proxies, timeout=_tiemout)
        html = html.content.decode(encoding)
        return html.replace('\x00', '').strip()
    except ChunkedEncodingError as e:
        html = get_stream(url, headers, encoding)
        return html
    except ConnectionError as e:
        return "ERROR:" + "HTTP连接错误"
    except ConnectTimeout as e:
        return "ERROR:" + "HTTP连接超时错误"
    except Exception as e:
        return 'ERROR:' + str(e) 

def test_watch_retry(self, m_sys_exit):
        path = '/test'
        events = [{'e': i} for i in range(3)]
        side_effects = [exceptions.ChunkedEncodingError("Connection Broken")]
        side_effects.extend(None for _ in events)

        m_handler = mock.Mock()
        m_handler.side_effect = side_effects
        watcher_obj = self._test_watch_create_watcher(path, m_handler, 10)
        self._test_watch_mock_events(watcher_obj, events)

        watcher_obj._watch(path)

        m_handler.assert_has_calls([mock.call(e) for e in events])
        m_sys_exit.assert_called_once_with(1) 

def main():
    count = 0
    while True:
        # print('第 ',count,' 次测试')
        count = count + 1
        try:
            #请求不同的代理和headers
            global headers,count_proxys
            headers = {'User-Agent': ua.random}
            count_proxys = get_count_proxys()
            print('代理总数： ',count_proxys,'  当前所用的代理：',proxy,'\n',headers)
            start_time = time.clock()
            html = crawl('http://www.baidu.com', proxy)
            end_time = time.clock()
            print('代理连接时间： ',(str(end_time-start_time))[:4],' 秒')
            if html.status_code==200:
                print(html)
                return count
                break
            elif count>=10:
                print('抓取网页失败')
                break

        except (ChunkedEncodingError,ConnectionError,Timeout,UnboundLocalError,UnicodeError,ProxyError):
            global proxy
            proxy = get_proxy()
            print('代理失败,更换代理','\n')
            # print(' ') 

def handle_stream(self, response):
        """
        Handles a stream of events from the Mastodon server. When each event
        is received, the corresponding .on_[name]() method is called.

        response; a requests response object with the open stream for reading.
        """
        event = {}
        line_buffer = bytearray()
        try:
            for chunk in response.iter_content(chunk_size = 1):
                if chunk:
                    for chunk_part in chunk:
                        chunk_part = bytearray([chunk_part])
                        if chunk_part == b'\n':
                            try:
                                line = line_buffer.decode('utf-8')
                            except UnicodeDecodeError as err:
                                exception = MastodonMalformedEventError("Malformed UTF-8")
                                self.on_abort(exception)
                                six.raise_from(
                                    exception,
                                    err
                                )
                            if line == '':
                                self._dispatch(event)
                                event = {}
                            else:
                                event = self._parse_line(line, event)
                            line_buffer = bytearray()
                        else:
                            line_buffer.extend(chunk_part)
        except ChunkedEncodingError as err:
            exception = MastodonNetworkError("Server ceased communication.")
            self.on_abort(exception)
            six.raise_from(
                exception,
                err
            )
        except MastodonReadTimeout as err:
            exception = MastodonReadTimeout("Timed out while reading from server."),
            self.on_abort(exception)
            six.raise_from(
                exception,
                err
            ) 

def check_fail(exception):
    """Checks for a requestslib exception, returns a signal if found.

    If this Exception is an instance of
    :class:`requests.exceptions.RequestException`, determine what kind of
    exception was raised. If not, return the results of from_generic_exception.

    :param Exception exception: An Exception object
    :returns: Signal with exception details
    :rtype: :class:`syntribos.signal.SynSignal`
    """
    check_name = "HTTP_CHECK_FAIL"

    def uncamel(string):
        string = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", string)
        return re.sub("([a-z0-9])([A-Z])", r"\1_\2", string).upper()

    if not isinstance(exception, rex.RequestException):
        return syntribos.signal.from_generic_exception(exception)

    data = {
        "response": exception.response,
        "request": exception.request,
        "exception": exception,
        "exception_name": uncamel(exception.__class__.__name__)
    }
    text = "An exception was encountered when sending the request. {desc}"
    slug = "HTTP_FAIL_{exc}".format(exc=data["exception_name"])
    tags = set(["EXCEPTION_RAISED"])

    invalid_request_exceptions = (rex.URLRequired, rex.MissingSchema,
                                  rex.InvalidSchema, rex.InvalidURL)

    if exception.__doc__:
        text = text.format(desc=exception.__doc__)
    else:
        text = text.format(
            desc="An unknown exception was raised. Please report this.")

    # CONNECTION FAILURES
    if isinstance(exception, (rex.ProxyError, rex.SSLError,
                              rex.ChunkedEncodingError, rex.ConnectionError)):
        tags.update(["CONNECTION_FAIL"])
    # TIMEOUTS
    elif isinstance(exception, (rex.ConnectTimeout, rex.ReadTimeout)):
        tags.update(["CONNECTION_TIMEOUT", "SERVER_FAIL"])
    # INVALID REQUESTS
    elif isinstance(exception, invalid_request_exceptions):
        tags.update(["INVALID_REQUEST", "CLIENT_FAIL"])

    return syntribos.signal.SynSignal(
        text=text,
        slug=slug,
        strength=1.0,
        tags=list(tags),
        data=data,
        check_name=check_name) 

def nitrx(code, headers, proxy):
    try:
        global running
        running += 1
    except:
        running = 0
    s = requests.session()
    s.proxies = proxy
    url = "https://discordapp.com/api/v6/entitlements/gift-codes/{}?with_application=false&with_subscription_plan=true".format(code)
    try:
        rr = s.get(url, headers=headers, timeout=config['timeout'])
        if "subscription_plan".lower() in (rr.text).lower():
            save(code)
            debug(code, "Valid", proxy, "Valid")
            running -= 1
            exit()
        o = json.loads(rr.text)
        message = o["message"].lower()
        if message == "Unknown Gift Code".lower():
            debug(code, "Invalid", proxy, 'Invalid')
        elif message == "You are being rate limited.".lower():
            debug(code, "Message", proxy, 'Message')
        elif message == "Access denied":
            debug(code, "Message", proxy, 'Message')
        else:
            print(rr.text)

    except KeyboardInterrupt:
        exit("[*] GoodBye")
    except ProxyError:
        debug(code, "Proxy", proxy, 'Error')
    except SSLError:
        debug(code, "SSL", proxy, 'Error')
    except ConnectionError:
        debug(code, "Connect", proxy, 'Error')
    except InvalidProxyURL:
        debug(code, "Proxy URL", proxy, 'Error')
    except requests.exceptions.ReadTimeout:
        debug(code, "Timeout", proxy, 'Error')
    except UnicodeError:
        debug(code, "UnicodeError", proxy, 'Error')
    except ChunkedEncodingError:
        debug(code, "Encoding", proxy, 'Error')
    except json.decoder.JSONDecodeError:
        debug(code, "J.Decode", proxy, 'Decode')

    running -= 1
    exit() 

def generate_proxied_request(self, url, method="GET", params={}, data={}, headers={}, req_timeout=30):
        try:
            random.shuffle(self.proxy_list)
            # req_headers = dict(params.items() + self.generate_random_request_headers().items())

            req_headers = dict(params.items())
            req_headers_random = dict(self.generate_random_request_headers().items())
            req_headers.update(req_headers_random)

            if not self.sustain:
                self.randomize_proxy()

            headers.update(req_headers)

            self.logger.debug("Using headers: {0}".format(str(headers)))
            self.logger.debug("Using proxy: {0}".format(str(self.current_proxy)))
            request = requests.request(method, url, headers=headers, data=data, params=params, timeout=req_timeout,
                    proxies={"http": self.current_proxy.get_address(), "https": self.current_proxy.get_address()})
            # Avoid HTTP request errors
            if request.status_code == 409:
                raise ConnectionError("HTTP Response [409] - Possible Cloudflare DNS resolution error")
            elif request.status_code == 403:
                raise ConnectionError("HTTP Response [403] - Permission denied error")
            elif request.status_code == 503:
                raise ConnectionError("HTTP Response [503] - Service unavailable error")
            self.logger.info('RR Status {}'.format(request.status_code))
            return request
        except ConnectionError:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Proxy unreachable - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()
        except ReadTimeout:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Read timed out - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()
        except ChunkedEncodingError:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Wrong server chunked encoding - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()
        except TooManyRedirects:
            try:
                self.proxy_list.remove(self.current_proxy)
            except ValueError:
                pass
            self.logger.debug("Too many redirects - Removed Straggling proxy: {0} PL Size = {1}".format(
                self.current_proxy, len(self.proxy_list)))
            self.randomize_proxy()