Python six.moves.http_client.HTTPResponse() Examples

def HTMLInputStream(source, encoding=None, parseMeta=True, chardet=True):
    if isinstance(source, http_client.HTTPResponse):
        # Work around Python bug #20007: read(0) closes the connection.
        # http://bugs.python.org/issue20007
        isUnicode = False
    elif hasattr(source, "read"):
        isUnicode = isinstance(source.read(0), text_type)
    else:
        isUnicode = isinstance(source, text_type)

    if isUnicode:
        if encoding is not None:
            raise TypeError("Cannot explicitly set an encoding with a unicode string")

        return HTMLUnicodeInputStream(source)
    else:
        return HTMLBinaryInputStream(source, encoding, parseMeta, chardet) 

def request(self, method, path, data, headers):
        req = webob.Request.blank(path)
        req.method = method
        req.body = data
        req.headers = headers
        req.headers['Accept'] = 'text/html'
        req.host = self.host
        # Call the WSGI app, get the HTTP response
        resp = str(req.get_response(self.app))
        # For some reason, the response doesn't have "HTTP/1.0 " prepended; I
        # guess that's a function the web server usually provides.
        resp = "HTTP/1.0 %s" % resp
        self.sock = FakeHttplibSocket(resp)
        self.http_response = http_client.HTTPResponse(self.sock)
        # NOTE(vish): boto is accessing private variables for some reason
        self._HTTPConnection__response = self.http_response
        self.http_response.begin() 

def request(self, method, path, body="", headers=None):
        """Translate request to WSGI app.

        Requests made via this connection actually get translated and routed
        into our WSGI app, we then wait for the response and turn it back into
        an `http_client.HTTPResponse`.
        """
        if not headers:
            headers = {}

        req = webob.Request.blank(path)
        req.method = method
        req.headers = headers
        req.host = self.host
        req.body = six.b(body)

        resp = str(req.get_response(self.app))
        resp = "HTTP/1.0 %s" % resp
        sock = FakeHttplibSocket(resp)
        self.http_response = http_client.HTTPResponse(sock)
        self.http_response.begin() 

def HTMLInputStream(source, encoding=None, parseMeta=True, chardet=True):
    if isinstance(source, http_client.HTTPResponse):
        # Work around Python bug #20007: read(0) closes the connection.
        # http://bugs.python.org/issue20007
        isUnicode = False
    elif hasattr(source, "read"):
        isUnicode = isinstance(source.read(0), text_type)
    else:
        isUnicode = isinstance(source, text_type)

    if isUnicode:
        if encoding is not None:
            raise TypeError("Cannot explicitly set an encoding with a unicode string")

        return HTMLUnicodeInputStream(source)
    else:
        return HTMLBinaryInputStream(source, encoding, parseMeta, chardet) 

def HTMLInputStream(source, encoding=None, parseMeta=True, chardet=True):
    if isinstance(source, http_client.HTTPResponse):
        # Work around Python bug #20007: read(0) closes the connection.
        # http://bugs.python.org/issue20007
        isUnicode = False
    elif hasattr(source, "read"):
        isUnicode = isinstance(source.read(0), text_type)
    else:
        isUnicode = isinstance(source, text_type)

    if isUnicode:
        if encoding is not None:
            raise TypeError("Cannot explicitly set an encoding with a unicode string")

        return HTMLUnicodeInputStream(source)
    else:
        return HTMLBinaryInputStream(source, encoding, parseMeta, chardet) 

def HTMLInputStream(source, encoding=None, parseMeta=True, chardet=True):
    if isinstance(source, http_client.HTTPResponse):
        # Work around Python bug #20007: read(0) closes the connection.
        # http://bugs.python.org/issue20007
        isUnicode = False
    elif hasattr(source, "read"):
        isUnicode = isinstance(source.read(0), text_type)
    else:
        isUnicode = isinstance(source, text_type)

    if isUnicode:
        if encoding is not None:
            raise TypeError("Cannot explicitly set an encoding with a unicode string")

        return HTMLUnicodeInputStream(source)
    else:
        return HTMLBinaryInputStream(source, encoding, parseMeta, chardet) 

def HTMLInputStream(source, **kwargs):
    # Work around Python bug #20007: read(0) closes the connection.
    # http://bugs.python.org/issue20007
    if (isinstance(source, http_client.HTTPResponse) or
        # Also check for addinfourl wrapping HTTPResponse
        (isinstance(source, urllib.response.addbase) and
         isinstance(source.fp, http_client.HTTPResponse))):
        isUnicode = False
    elif hasattr(source, "read"):
        isUnicode = isinstance(source.read(0), text_type)
    else:
        isUnicode = isinstance(source, text_type)

    if isUnicode:
        encodings = [x for x in kwargs if x.endswith("_encoding")]
        if encodings:
            raise TypeError("Cannot set an encoding with a unicode input, set %r" % encodings)

        return HTMLUnicodeInputStream(source, **kwargs)
    else:
        return HTMLBinaryInputStream(source, **kwargs) 

def test_python_issue_20007_b():
    """
    Make sure we have a work-around for Python bug #20007
    http://bugs.python.org/issue20007
    """
    if six.PY2:
        return

    class FakeSocket(object):
        def makefile(self, _mode, _bufsize=None):
            # pylint:disable=unused-argument
            return BytesIO(b"HTTP/1.1 200 Ok\r\n\r\nText")

    source = http_client.HTTPResponse(FakeSocket())
    source.begin()
    wrapped = urllib.response.addinfourl(source, source.msg, "http://example.com")
    stream = HTMLInputStream(wrapped)
    assert stream.charsUntil(" ") == "Text" 

def HTMLInputStream(source, encoding=None, parseMeta=True, chardet=True):
    if isinstance(source, http_client.HTTPResponse):
        # Work around Python bug #20007: read(0) closes the connection.
        # http://bugs.python.org/issue20007
        isUnicode = False
    elif hasattr(source, "read"):
        isUnicode = isinstance(source.read(0), text_type)
    else:
        isUnicode = isinstance(source, text_type)

    if isUnicode:
        if encoding is not None:
            raise TypeError("Cannot explicitly set an encoding with a unicode string")

        return HTMLUnicodeInputStream(source)
    else:
        return HTMLBinaryInputStream(source, encoding, parseMeta, chardet) 

def close(self):
        ''' Close the connection to server.

        If available, return a http_client.HTTPResponse object.

        Closing the connection involves sending the
        Transfer-Encoding terminating bytes.
        '''
        self._reset_retries()
        self._closed = True

        # Chunked-encoded posts are terminated with '0\r\n\r\n'
        # For some reason, either Python or node.js seems to
        # require an extra \r\n.
        try:
            self._conn.send('\r\n0\r\n\r\n'.encode('utf-8'))
        except http_client.socket.error:
            # In case the socket has already been closed
            return ''

        return self._getresponse() 

def __init__(self, recorder, *args, **kwargs):
        httplib.HTTPResponse.__init__(self, *args, **kwargs)
        self.fp = RecordingStream(self.fp, recorder)


# ============================================================================ 

def test_python_issue_20007(self):
        """
        Make sure we have a work-around for Python bug #20007
        http://bugs.python.org/issue20007
        """
        class FakeSocket(object):
            def makefile(self, _mode, _bufsize=None):
                return BytesIO(b"HTTP/1.1 200 Ok\r\n\r\nText")

        source = http_client.HTTPResponse(FakeSocket())
        source.begin()
        stream = HTMLInputStream(source)
        self.assertEqual(stream.charsUntil(" "), "Text") 

def test_python_issue_20007():
    """
    Make sure we have a work-around for Python bug #20007
    http://bugs.python.org/issue20007
    """
    class FakeSocket(object):
        def makefile(self, _mode, _bufsize=None):
            # pylint:disable=unused-argument
            return BytesIO(b"HTTP/1.1 200 Ok\r\n\r\nText")

    source = http_client.HTTPResponse(FakeSocket())
    source.begin()
    stream = HTMLInputStream(source)
    assert stream.charsUntil(" ") == "Text" 

def _getresponse(self):
        ''' Read from recv and return a HTTPResponse object if possible.
        Either
        1 - The client has succesfully closed the connection: Return ''
        2 - The server has already closed the connection: Return the response
            if possible.
        '''
        # Wait for a response
        self._conn.sock.setblocking(True)
        # Parse the response
        response = self._bytes
        while True:
            try:
                _bytes = self._conn.sock.recv(1)
            except http_client.socket.error:
                # For error 54: Connection reset by peer
                # (and perhaps others)
                return six.b('')
            if _bytes == six.b(''):
                break
            else:
                response += _bytes
        # Set recv to be non-blocking again
        self._conn.sock.setblocking(False)

        # Convert the response string to a http_client.HTTPResponse
        # object with a bit of a hack
        if response != six.b(''):
            # Taken from
            # http://pythonwise.blogspot.ca/2010/02/parse-http-response.html
            try:
                response = http_client.HTTPResponse(_FakeSocket(response))
                response.begin()
            except:
                # Bad headers ... etc.
                response = six.b('')
        return response 

def open_file(
    link,  # type: Union[_T, str]
    session=None,  # type: Optional[Session]
    stream=True,  # type: bool
):
    # type: (...) -> ContextManager[Union[IO[bytes], Urllib3_HTTPResponse, Urllib_HTTPResponse]]
    """
    Open local or remote file for reading.

    :param pip._internal.index.Link link: A link object from resolving dependencies with
        pip, or else a URL.
    :param Optional[Session] session: A :class:`~requests.Session` instance
    :param bool stream: Whether to stream the content if remote, default True
    :raises ValueError: If link points to a local directory.
    :return: a context manager to the opened file-like object
    """
    if not isinstance(link, six.string_types):
        try:
            link = link.url_without_fragment
        except AttributeError:
            raise ValueError("Cannot parse url from unkown type: {0!r}".format(link))

    if not is_valid_url(link) and os.path.exists(link):
        link = path_to_url(link)

    if is_file_url(link):
        # Local URL
        local_path = url_to_path(link)
        if os.path.isdir(local_path):
            raise ValueError("Cannot open directory for read: {}".format(link))
        else:
            with io.open(local_path, "rb") as local_file:
                yield local_file
    else:
        # Remote URL
        headers = {"Accept-Encoding": "identity"}
        if not session:
            try:
                from requests import Session  # noqa
            except ImportError:
                session = None
            else:
                session = Session()
        if session is None:
            with closing(six.moves.urllib.request.urlopen(link)) as f:
                yield f
        else:
            with session.get(link, headers=headers, stream=stream) as resp:
                try:
                    raw = getattr(resp, "raw", None)
                    result = raw if raw else resp
                    yield result
                finally:
                    if raw:
                        conn = raw._connection
                        if conn is not None:
                            conn.close()
                    result.close() 

def write(self, data, reconnect_on=('', 200, )):
        ''' Send `data` to the server in chunk-encoded form.
        Check the connection before writing and reconnect
        if disconnected and if the response status code is in `reconnect_on`.

        The response may either be an HTTPResponse object or an empty string.
        '''

        if not self._isconnected():

            # Attempt to get the response.
            response = self._getresponse()

            # Reconnect depending on the status code.
            if ((response == '' and '' in reconnect_on) or
                (response and isinstance(response, http_client.HTTPResponse) and
                 response.status in reconnect_on)):
                self._reconnect()

            elif response and isinstance(response, http_client.HTTPResponse):
                # If an HTTPResponse was recieved then
                # make the users aware instead of
                # auto-reconnecting in case the
                # server is responding with an important
                # message that might prevent
                # future requests from going through,
                # like Invalid Credentials.
                # This allows the user to determine when
                # to reconnect.
                raise Exception("Server responded with "
                                "status code: {status_code}\n"
                                "and message: {msg}."
                                .format(status_code=response.status,
                                        msg=response.read()))

            elif response == '':
                raise Exception("Attempted to write but socket "
                                "was not connected.")

        try:
            msg = data
            msglen = format(len(msg), 'x')  # msg length in hex
            # Send the message in chunk-encoded form
            self._conn.sock.setblocking(1)
            self._conn.send('{msglen}\r\n{msg}\r\n'
                            .format(msglen=msglen, msg=msg).encode('utf-8'))
            self._conn.sock.setblocking(0)
        except http_client.socket.error:
            self._reconnect()
            self.write(data) 

def open_file(
    link,  # type: Union[_T, str]
    session=None,  # type: Optional[Session]
    stream=True,  # type: bool
):
    # type: (...) -> ContextManager[Union[IO[bytes], Urllib3_HTTPResponse, Urllib_HTTPResponse]]
    """
    Open local or remote file for reading.

    :param pip._internal.index.Link link: A link object from resolving dependencies with
        pip, or else a URL.
    :param Optional[Session] session: A :class:`~requests.Session` instance
    :param bool stream: Whether to stream the content if remote, default True
    :raises ValueError: If link points to a local directory.
    :return: a context manager to the opened file-like object
    """
    if not isinstance(link, six.string_types):
        try:
            link = link.url_without_fragment
        except AttributeError:
            raise ValueError("Cannot parse url from unkown type: {0!r}".format(link))

    if not is_valid_url(link) and os.path.exists(link):
        link = path_to_url(link)

    if is_file_url(link):
        # Local URL
        local_path = url_to_path(link)
        if os.path.isdir(local_path):
            raise ValueError("Cannot open directory for read: {}".format(link))
        else:
            with io.open(local_path, "rb") as local_file:
                yield local_file
    else:
        # Remote URL
        headers = {"Accept-Encoding": "identity"}
        if not session:
            try:
                from requests import Session  # noqa
            except ImportError:
                session = None
            else:
                session = Session()
        if session is None:
            with closing(six.moves.urllib.request.urlopen(link)) as f:
                yield f
        else:
            with session.get(link, headers=headers, stream=stream) as resp:
                try:
                    raw = getattr(resp, "raw", None)
                    result = raw if raw else resp
                    yield result
                finally:
                    if raw:
                        conn = raw._connection
                        if conn is not None:
                            conn.close()
                    result.close()