Python twisted.web.client.HTTPClientFactory() Examples

def makeRequest(self, path, method, headers, body):
        scheme = "https:" if self.useSSL else "http:"
        url = "%s//%s:%d%s" % (scheme, self.host, self.port, path)
        caldavFactory = client.HTTPClientFactory(
            url, method=method,
            headers=headers, postdata=body, agent="Push Monitor")
        caldavFactory.username = self.authname
        caldavFactory.password = self.password
        caldavFactory.noisy = False
        caldavFactory.protocol = PropfindRequestor
        if self.useSSL:
            connect(GAIEndpoint(reactor, self.host, self.port, simpleClientContextFactory(self.host)),
                    caldavFactory)
        else:
            connect(GAIEndpoint(reactor, self.host, self.port), caldavFactory)

        return caldavFactory.deferred 

def test_afterFoundGet(self):
        """
        Enabling unsafe redirection behaviour overwrites the method of
        redirected C{POST} requests with C{GET}.
        """
        url = self.getURL('extendedRedirect?code=302')
        f = client.HTTPClientFactory(url, followRedirect=True, method="POST")
        self.assertFalse(
            f.afterFoundGet,
            "By default, afterFoundGet must be disabled")

        def gotPage(page):
            self.assertEquals(
                self.extendedRedirect.lastMethod,
                "GET",
                "With afterFoundGet, the HTTP method must change to GET")

        d = client.getPage(
            url, followRedirect=True, afterFoundGet=True, method="POST")
        d.addCallback(gotPage)
        return d 

def test_infiniteRedirection(self):
        """
        When more than C{redirectLimit} HTTP redirects are encountered, the
        page request fails with L{InfiniteRedirection}.
        """
        def checkRedirectCount(*a):
            self.assertEqual(f._redirectCount, 13)
            self.assertEqual(self.infiniteRedirectResource.count, 13)

        f = client._makeGetterFactory(
            self.getURL('infiniteRedirect'),
            client.HTTPClientFactory,
            redirectLimit=13)
        d = self.assertFailure(f.deferred, error.InfiniteRedirection)
        d.addCallback(checkRedirectCount)
        return d 

def test_protectedServerAndDate(self):
        """
        If the CGI script emits a I{Server} or I{Date} header, these are
        ignored.
        """
        cgiFilename = self.writeCGI(SPECIAL_HEADER_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)
        factory = client.HTTPClientFactory(url)
        reactor.connectTCP('localhost', portnum, factory)
        def checkResponse(ignored):
            self.assertNotIn('monkeys', factory.response_headers['server'])
            self.assertNotIn('last year', factory.response_headers['date'])
        factory.deferred.addCallback(checkResponse)
        return factory.deferred 

def test_duplicateHeaderCGI(self):
        """
        If a CGI script emits two instances of the same header, both are sent in
        the response.
        """
        cgiFilename = self.writeCGI(DUAL_HEADER_CGI)

        portnum = self.startServer(cgiFilename)
        url = "http://localhost:%d/cgi" % (portnum,)
        factory = client.HTTPClientFactory(url)
        reactor.connectTCP('localhost', portnum, factory)
        def checkResponse(ignored):
            self.assertEquals(
                factory.response_headers['header'], ['spam', 'eggs'])
        factory.deferred.addCallback(checkResponse)
        return factory.deferred 

def test_afterFoundGet(self):
        """
        Enabling unsafe redirection behaviour overwrites the method of
        redirected C{POST} requests with C{GET}.
        """
        url = self.getURL('extendedRedirect?code=302')
        f = client.HTTPClientFactory(url, followRedirect=True, method=b"POST")
        self.assertFalse(
            f.afterFoundGet,
            "By default, afterFoundGet must be disabled")

        def gotPage(page):
            self.assertEqual(
                self.extendedRedirect.lastMethod,
                b"GET",
                "With afterFoundGet, the HTTP method must change to GET")

        d = client.getPage(
            url, followRedirect=True, afterFoundGet=True, method=b"POST")
        d.addCallback(gotPage)
        return d 

def test_infiniteRedirection(self):
        """
        When more than C{redirectLimit} HTTP redirects are encountered, the
        page request fails with L{InfiniteRedirection}.
        """
        def checkRedirectCount(*a):
            self.assertEquals(f._redirectCount, 13)
            self.assertEquals(self.infiniteRedirectResource.count, 13)

        f = client._makeGetterFactory(
            self.getURL('infiniteRedirect'),
            client.HTTPClientFactory,
            redirectLimit=13)
        d = self.assertFailure(f.deferred, error.InfiniteRedirection)
        d.addCallback(checkRedirectCount)
        return d 

def _getPage(url, descriptor):
    """
    Fetch the body of the given url via HTTP, connecting to the given host
    and port.

    @param url: The URL to GET
    @type url: C{str}
    @param descriptor: The endpoint descriptor to use
    @type descriptor: C{str}
    @return: A deferred; upon 200 success the body of the response is returned,
        otherwise a twisted.web.error.Error is the result.
    """
    point = endpoints.clientFromString(reactor, descriptor)
    factory = HTTPClientFactory(url, timeout=10)
    point.connect(factory)
    return factory.deferred 

def test_afterFoundGet(self):
        """
        Enabling unsafe redirection behaviour overwrites the method of
        redirected C{POST} requests with C{GET}.
        """
        url = self.getURL('extendedRedirect?code=302')
        f = client.HTTPClientFactory(url, followRedirect=True, method=b"POST")
        self.assertFalse(
            f.afterFoundGet,
            "By default, afterFoundGet must be disabled")

        def gotPage(page):
            self.assertEqual(
                self.extendedRedirect.lastMethod,
                b"GET",
                "With afterFoundGet, the HTTP method must change to GET")

        d = client.getPage(
            url, followRedirect=True, afterFoundGet=True, method=b"POST")
        d.addCallback(gotPage)
        return d 

def test_infiniteRedirection(self):
        """
        When more than C{redirectLimit} HTTP redirects are encountered, the
        page request fails with L{InfiniteRedirection}.
        """
        def checkRedirectCount(*a):
            self.assertEqual(f._redirectCount, 13)
            self.assertEqual(self.infiniteRedirectResource.count, 13)

        f = client._makeGetterFactory(
            self.getURL('infiniteRedirect'),
            client.HTTPClientFactory,
            redirectLimit=13)
        d = self.assertFailure(f.deferred, error.InfiniteRedirection)
        d.addCallback(checkRedirectCount)
        return d 

def test_httpClientFactoryDeprecated(self):
        """
        L{client.HTTPClientFactory} is deprecated.
        """
        self._testDeprecatedClass("HTTPClientFactory") 

def test_earlyHeaders(self):
        """
        When a connection is made, L{HTTPPagerGetter} sends the headers from
        its factory's C{headers} dict.  If I{Host} or I{Content-Length} is
        present in this dict, the values are not sent, since they are sent with
        special values before the C{headers} dict is processed.  If
        I{User-Agent} is present in the dict, it overrides the value of the
        C{agent} attribute of the factory.  If I{Cookie} is present in the
        dict, its value is added to the values from the factory's C{cookies}
        attribute.
        """
        factory = client.HTTPClientFactory(
            'http://foo/bar',
            agent="foobar",
            cookies={'baz': 'quux'},
            postdata="some data",
            headers={
                'Host': 'example.net',
                'User-Agent': 'fooble',
                'Cookie': 'blah blah',
                'Content-Length': '12981',
                'Useful': 'value'})
        transport = StringTransport()
        protocol = client.HTTPPageGetter()
        protocol.factory = factory
        protocol.makeConnection(transport)
        self.assertEqual(
            transport.value(),
            "GET /bar HTTP/1.0\r\n"
            "Host: example.net\r\n"
            "User-Agent: foobar\r\n"
            "Content-Length: 9\r\n"
            "Useful: value\r\n"
            "connection: close\r\n"
            "Cookie: blah blah; baz=quux\r\n"
            "\r\n"
            "some data") 

def testFactoryInfo(self):
        url = self.getURL('file')
        scheme, host, port, path = client._parse(url)
        factory = client.HTTPClientFactory(url)
        reactor.connectTCP(host, port, factory)
        return factory.deferred.addCallback(self._cbFactoryInfo, factory) 

def __init__(self, request, timeout=180):
        self._url = urldefrag(request.url)[0]
        # converting to bytes to comply to Twisted interface
        self.url = to_bytes(self._url, encoding='ascii')
        self.method = to_bytes(request.method, encoding='ascii')
        self.body = request.body or None
        self.headers = Headers(request.headers)
        self.response_headers = None
        self.timeout = request.meta.get('download_timeout') or timeout
        self.start_time = time()
        self.deferred = defer.Deferred().addCallback(self._build_response, request)

        # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected
        # to have _disconnectedDeferred. See Twisted r32329.
        # As Scrapy implements it's own logic to handle redirects is not
        # needed to add the callback _waitForDisconnect.
        # Specifically this avoids the AttributeError exception when
        # clientConnectionFailed method is called.
        self._disconnectedDeferred = defer.Deferred()

        self._set_connection_attributes(request)

        # set Host header based on url
        self.headers.setdefault('Host', self.netloc)

        # set Content-Length based len of body
        if self.body is not None:
            self.headers['Content-Length'] = len(self.body)
            # just in case a broken http/1.1 decides to keep connection alive
            self.headers.setdefault("Connection", "close")
        # Content-Length must be specified in POST method even with no body
        elif self.method == b'POST':
            self.headers['Content-Length'] = 0 

def attemptRequestWithMaliciousURI(self, uri):
        """
        Attempt a request with the provided URI.

        @param uri: L{URIInjectionTestsMixin}
        """
        client.HTTPClientFactory(b"https://twisted.invalid").setURL(uri) 

def attemptRequestWithMaliciousURI(self, uri):
        """
        Attempt a request with the provided URI.

        @param uri: L{URIInjectionTestsMixin}
        """
        client.HTTPClientFactory(uri) 

def test_HTTPSDefaultPort(self):
        """
        No port should be included in the host header when connecting to the
        default HTTPS port.
        """
        factory = client.HTTPClientFactory(b'https://foo.example.com/')
        proto = factory.buildProtocol('127.42.42.42')
        proto.makeConnection(StringTransport())
        self.assertEqual(self._getHost(proto.transport.value()),
                          b'foo.example.com') 

def testFactoryInfo(self):
        url = self.getURL('file')
        scheme, host, port, path = client._parse(url)
        factory = client.HTTPClientFactory(url)
        reactor.connectSSL(host, port, factory, ssl.ClientContextFactory())
        # The base class defines _cbFactoryInfo correctly for this
        return factory.deferred.addCallback(self._cbFactoryInfo, factory) 

def __init__(self, url, method='GET', postdata=None, headers=None,
                 agent="Twisted PageGetter", timeout=0, cookies=None,
                 followRedirect=1, proxy=None):
        if headers is None:
            headers = {}
        headers['Accept-encoding'] = 'gzip'
        self.proxy = proxy
        client.HTTPClientFactory.__init__(self, url, method=method,
                                          postdata=postdata, headers=headers,
                                          agent=agent, timeout=timeout,
                                          cookies=cookies,
                                          followRedirect=followRedirect) 

def setURL(self, url):
        client.HTTPClientFactory.setURL(self, url)
        if self.proxy:
            self.path = "%s://%s:%s%s" % (self.scheme,  
                                          self.host,  
                                          self.port,  
                                          self.path) 

def getPageFactory(url,
                   agent="BitTorrent client",
                   bindAddress=None,
                   contextFactory=None,
                   proxy=None,
                   timeout=120):
    """Download a web page as a string.

    Download a page. Return a deferred, which will callback with a
    page (as a string) or errback with a description of the error.

    See HTTPClientFactory to see what extra args can be passed.
    """
    scheme, host, port, path = client._parse(url)
    if proxy:
        host, port = proxy.split(':')
        port = int(port)
    factory = HTTPProxyUnGzipClientFactory(url, agent=agent, proxy=proxy)
    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory,
                           bindAddress=bindAddress,
                           timeout=timeout)
    else:
        reactor.connectTCP(host, port, factory,
                           bindAddress=bindAddress,
                           timeout=timeout)
    return factory 

def _getPage(self):
        factory = client.HTTPClientFactory(self.proxyHost, self.url)
        factory.headers = {'pragma': 'no-cache'}
        reactor.connectTCP(self.proxyHost, self.proxyPort, factory)
        d = factory.deferred
        d.addErrback(self.noPage)
        d.addCallback(self.page) 

def testFactoryInfo(self):
        url = self.getURL('file')
        scheme, host, port, path = client._parse(url)
        factory = client.HTTPClientFactory(url)
        reactor.connectTCP(host, port, factory)
        return factory.deferred.addCallback(self._cbFactoryInfo, factory) 

def testCookieHeaderParsing(self):
        d = defer.Deferred()
        factory = client.HTTPClientFactory('http://foo.example.com/')
        proto = factory.buildProtocol('127.42.42.42')
        proto.transport = FakeTransport()
        proto.connectionMade()
        for line in [
            '200 Ok',
            'Squash: yes',
            'Hands: stolen',
            'Set-Cookie: CUSTOMER=WILE_E_COYOTE; path=/; expires=Wednesday, 09-Nov-99 23:12:40 GMT',
            'Set-Cookie: PART_NUMBER=ROCKET_LAUNCHER_0001; path=/',
            'Set-Cookie: SHIPPING=FEDEX; path=/foo',
            '',
            'body',
            'more body',
            ]:
            proto.dataReceived(line + '\r\n')
        self.assertEquals(proto.transport.data,
                          ['GET / HTTP/1.0\r\n',
                           'Host: foo.example.com\r\n',
                           'User-Agent: Twisted PageGetter\r\n',
                           '\r\n'])
        self.assertEquals(factory.cookies,
                          {
            'CUSTOMER': 'WILE_E_COYOTE',
            'PART_NUMBER': 'ROCKET_LAUNCHER_0001',
            'SHIPPING': 'FEDEX',
            }) 

def test_request_url_http(self, r):
        url = 'http://google.com/'
        host = None

        request_url(url, host)
        ct = r.connectTCP

        self.assertEquals(ct.call_count, 1)
        args, kwargs = ct.call_args
        self.assertEquals(args[0], 'google.com')
        self.assertEquals(args[1], 80)
        self.assertTrue(isinstance(args[2], client.HTTPClientFactory)) 

def test_request_url_ssl(self, r):
        url = 'https://google.com/'
        host = None

        request_url(url, host)
        cs = r.connectSSL

        self.assertEquals(cs.call_count, 1)
        args, kwargs = cs.call_args
        self.assertEquals(args[0], 'google.com')
        self.assertEquals(args[1], 443)
        self.assertTrue(isinstance(args[2], client.HTTPClientFactory))
        self.assertTrue(isinstance(args[3], ssl.ClientContextFactory)) 

def getPage(url, contextFactory=None, *args, **kwargs):
    """Download a web page as a string.

    Download a page. Return a HTTPClientFactory

    See HTTPClientFactory to see what extra args can be passed.
    """
    #scheme, host, port, path = client._parse(url)
    scheme, _ = url.split('://', 1)
    
    host_port, path = _.split('/', 1)
    try:
        host, port = host_port.split(':')
        port = int(port)
    except Exception:
        host = host_port
        port = 80
    path = '/'+path
    factory = client.HTTPClientFactory(url, *args, **kwargs)
    factory.noisy = False
    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)
    return factory

### files usage 

def send_json(url, **kw):
    qurl =url+'?'+urlencode(kw.get('postdata', {}))
    # debug(0,'URLSTATS: %s',qurl)
    factory = HTTPClientFactory(str(qurl))
    factory.noisy = False
    if url.startswith('http://'):
        reactor.connectTCP(factory.host, factory.port, factory)
    elif url.startswith('https://'):
        reactor.connectSSL(factory.host, factory.port, factory, 
            ssl.ClientContextFactory())
    else:
        raise Exception('Url error: %s' %url)
    return factory 

def __init__(self, request, timeout=180):
        self._url = urldefrag(request.url)[0]
        # converting to bytes to comply to Twisted interface
        self.url = to_bytes(self._url, encoding='ascii')
        self.method = to_bytes(request.method, encoding='ascii')
        self.body = request.body or None
        self.headers = Headers(request.headers)
        self.response_headers = None
        self.timeout = request.meta.get('download_timeout') or timeout
        self.start_time = time()
        self.deferred = defer.Deferred().addCallback(self._build_response, request)

        # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected
        # to have _disconnectedDeferred. See Twisted r32329.
        # As Scrapy implements it's own logic to handle redirects is not
        # needed to add the callback _waitForDisconnect.
        # Specifically this avoids the AttributeError exception when
        # clientConnectionFailed method is called.
        self._disconnectedDeferred = defer.Deferred()

        self._set_connection_attributes(request)

        # set Host header based on url
        self.headers.setdefault('Host', self.netloc)

        # set Content-Length based len of body
        if self.body is not None:
            self.headers['Content-Length'] = len(self.body)
            # just in case a broken http/1.1 decides to keep connection alive
            self.headers.setdefault("Connection", "close")
        # Content-Length must be specified in POST method even with no body
        elif self.method == b'POST':
            self.headers['Content-Length'] = 0 

def testFactoryInfo(self):
        url = self.getURL('file')
        uri = client.URI.fromBytes(url)
        factory = client.HTTPClientFactory(url)
        reactor.connectTCP(nativeString(uri.host), uri.port, factory)
        return factory.deferred.addCallback(self._cbFactoryInfo, factory)