Python pycurl.CurlMulti() Examples
The following are 12
code examples of pycurl.CurlMulti().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
pycurl
, or try the search function
.
Example #1
Source File: httpclient.py From honeything with GNU General Public License v3.0 | 6 votes |
def __new__(cls, io_loop=None, max_clients=10, max_simultaneous_connections=None): # There is one client per IOLoop since they share curl instances io_loop = io_loop or ioloop.IOLoop.instance() if io_loop in cls._ASYNC_CLIENTS: return cls._ASYNC_CLIENTS[io_loop] else: instance = super(AsyncHTTPClient, cls).__new__(cls) instance.io_loop = io_loop instance._multi = pycurl.CurlMulti() instance._curls = [_curl_create(max_simultaneous_connections) for i in xrange(max_clients)] instance._free_list = instance._curls[:] instance._requests = collections.deque() instance._fds = {} instance._events = {} instance._added_perform_callback = False instance._timeout = None instance._closed = False cls._ASYNC_CLIENTS[io_loop] = instance return instance
Example #2
Source File: httpclient.py From honeything with GNU General Public License v3.0 | 6 votes |
def __new__(cls, io_loop=None, max_clients=10, max_simultaneous_connections=None): # There is one client per IOLoop since they share curl instances io_loop = io_loop or ioloop.IOLoop.instance() if io_loop in cls._ASYNC_CLIENTS: return cls._ASYNC_CLIENTS[io_loop] else: instance = super(AsyncHTTPClient2, cls).__new__(cls) instance.io_loop = io_loop instance._multi = pycurl.CurlMulti() instance._multi.setopt(pycurl.M_TIMERFUNCTION, instance._set_timeout) instance._multi.setopt(pycurl.M_SOCKETFUNCTION, instance._handle_socket) instance._curls = [_curl_create(max_simultaneous_connections) for i in xrange(max_clients)] instance._free_list = instance._curls[:] instance._requests = collections.deque() instance._fds = {} instance._timeout = None cls._ASYNC_CLIENTS[io_loop] = instance return instance
Example #3
Source File: myhttp.py From wfuzz with GNU General Public License v2.0 | 6 votes |
def _initialize(self): # pycurl Connection pool self.m = pycurl.CurlMulti() self.handles = [] for i in range(self.options.get("concurrent")): curl_h = pycurl.Curl() self.handles.append(curl_h) self.curlh_freelist.append(curl_h) # create threads self.ths = [] for fn in ("_read_multi_stack",): th = Thread(target=getattr(self, fn)) th.setName(fn) self.ths.append(th) th.start()
Example #4
Source File: curl_httpclient.py From tornado-zh with MIT License | 5 votes |
def initialize(self, io_loop, max_clients=10, defaults=None): super(CurlAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) self._curls = [self._curl_create() for i in range(max_clients)] self._free_list = self._curls[:] self._requests = collections.deque() self._fds = {} self._timeout = None # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by # forcing a periodic scan of all active requests. self._force_timeout_callback = ioloop.PeriodicCallback( self._handle_force_timeout, 1000, io_loop=io_loop) self._force_timeout_callback.start() # Work around a bug in libcurl 7.29.0: Some fields in the curl # multi object are initialized lazily, and its destructor will # segfault if it is destroyed without having been used. Add # and remove a dummy handle to make sure everything is # initialized. dummy_curl_handle = pycurl.Curl() self._multi.add_handle(dummy_curl_handle) self._multi.remove_handle(dummy_curl_handle)
Example #5
Source File: curl_httpclient.py From tornado-zh with MIT License | 5 votes |
def initialize(self, io_loop, max_clients=10, defaults=None): super(CurlAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) self._curls = [self._curl_create() for i in range(max_clients)] self._free_list = self._curls[:] self._requests = collections.deque() self._fds = {} self._timeout = None # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by # forcing a periodic scan of all active requests. self._force_timeout_callback = ioloop.PeriodicCallback( self._handle_force_timeout, 1000, io_loop=io_loop) self._force_timeout_callback.start() # Work around a bug in libcurl 7.29.0: Some fields in the curl # multi object are initialized lazily, and its destructor will # segfault if it is destroyed without having been used. Add # and remove a dummy handle to make sure everything is # initialized. dummy_curl_handle = pycurl.Curl() self._multi.add_handle(dummy_curl_handle) self._multi.remove_handle(dummy_curl_handle)
Example #6
Source File: curl_httpclient.py From viewfinder with Apache License 2.0 | 5 votes |
def initialize(self, io_loop, max_clients=10, defaults=None): super(CurlAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) self._curls = [_curl_create() for i in range(max_clients)] self._free_list = self._curls[:] self._requests = collections.deque() self._fds = {} self._timeout = None try: self._socket_action = self._multi.socket_action except AttributeError: # socket_action is found in pycurl since 7.18.2 (it's been # in libcurl longer than that but wasn't accessible to # python). gen_log.warning("socket_action method missing from pycurl; " "falling back to socket_all. Upgrading " "libcurl and pycurl will improve performance") self._socket_action = \ lambda fd, action: self._multi.socket_all() # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by # forcing a periodic scan of all active requests. self._force_timeout_callback = ioloop.PeriodicCallback( self._handle_force_timeout, 1000, io_loop=io_loop) self._force_timeout_callback.start() # Work around a bug in libcurl 7.29.0: Some fields in the curl # multi object are initialized lazily, and its destructor will # segfault if it is destroyed without having been used. Add # and remove a dummy handle to make sure everything is # initialized. dummy_curl_handle = pycurl.Curl() self._multi.add_handle(dummy_curl_handle) self._multi.remove_handle(dummy_curl_handle)
Example #7
Source File: curl_httpclient.py From viewfinder with Apache License 2.0 | 5 votes |
def initialize(self, io_loop, max_clients=10, defaults=None): super(CurlAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) self._curls = [_curl_create() for i in range(max_clients)] self._free_list = self._curls[:] self._requests = collections.deque() self._fds = {} self._timeout = None try: self._socket_action = self._multi.socket_action except AttributeError: # socket_action is found in pycurl since 7.18.2 (it's been # in libcurl longer than that but wasn't accessible to # python). gen_log.warning("socket_action method missing from pycurl; " "falling back to socket_all. Upgrading " "libcurl and pycurl will improve performance") self._socket_action = \ lambda fd, action: self._multi.socket_all() # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by # forcing a periodic scan of all active requests. self._force_timeout_callback = ioloop.PeriodicCallback( self._handle_force_timeout, 1000, io_loop=io_loop) self._force_timeout_callback.start() # Work around a bug in libcurl 7.29.0: Some fields in the curl # multi object are initialized lazily, and its destructor will # segfault if it is destroyed without having been used. Add # and remove a dummy handle to make sure everything is # initialized. dummy_curl_handle = pycurl.Curl() self._multi.add_handle(dummy_curl_handle) self._multi.remove_handle(dummy_curl_handle)
Example #8
Source File: pycurldownload.py From QMusic with GNU Lesser General Public License v2.1 | 5 votes |
def __init__(self): file(DLOG, 'w') # logging.basicConfig(level=LOGLEVEL, # format='[% (asctime)s][%(levelname)s] % (message)s', # filename='download.log', # filenmode='w') self.mcurl = pycurl.CurlMulti()
Example #9
Source File: curl_httpclient.py From honeything with GNU General Public License v3.0 | 5 votes |
def initialize(self, io_loop=None, max_clients=10, max_simultaneous_connections=None): self.io_loop = io_loop self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) self._curls = [_curl_create(max_simultaneous_connections) for i in xrange(max_clients)] self._free_list = self._curls[:] self._requests = collections.deque() self._fds = {} self._timeout = None try: self._socket_action = self._multi.socket_action except AttributeError: # socket_action is found in pycurl since 7.18.2 (it's been # in libcurl longer than that but wasn't accessible to # python). logging.warning("socket_action method missing from pycurl; " "falling back to socket_all. Upgrading " "libcurl and pycurl will improve performance") self._socket_action = \ lambda fd, action: self._multi.socket_all() # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by # forcing a periodic scan of all active requests. self._force_timeout_callback = ioloop.PeriodicCallback( self._handle_force_timeout, 1000, io_loop=io_loop) self._force_timeout_callback.start()
Example #10
Source File: curl_httpclient.py From EventGhost with GNU General Public License v2.0 | 5 votes |
def initialize(self, io_loop, max_clients=10, defaults=None): super(CurlAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) self._curls = [self._curl_create() for i in range(max_clients)] self._free_list = self._curls[:] self._requests = collections.deque() self._fds = {} self._timeout = None # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by # forcing a periodic scan of all active requests. self._force_timeout_callback = ioloop.PeriodicCallback( self._handle_force_timeout, 1000, io_loop=io_loop) self._force_timeout_callback.start() # Work around a bug in libcurl 7.29.0: Some fields in the curl # multi object are initialized lazily, and its destructor will # segfault if it is destroyed without having been used. Add # and remove a dummy handle to make sure everything is # initialized. dummy_curl_handle = pycurl.Curl() self._multi.add_handle(dummy_curl_handle) self._multi.remove_handle(dummy_curl_handle)
Example #11
Source File: client.py From pycopia with Apache License 2.0 | 4 votes |
def perform(self, logfile=None): """Fetch all of the added Request objects. Return two lists. The first list is a list of responses that completed. The second is list of the requests that errored. """ m = pycurl.CurlMulti() requests = [] num_q = num_urls = len(self._requests) reqs = self._requests self._requests = [] for req in reqs: c, resp = req.get_requester() c.resp = resp m.add_handle(c) requests.append(c) del reqs while 1: ret, num_handles = m.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: break num_handles = num_urls while num_handles: ret = m.select(5.0) if ret == -1: continue while 1: ret, num_handles = m.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: break goodlist = [] errlist = [] while 1: num_q, ok_list, err_list = m.info_read(num_urls) for c in ok_list: resp = c.resp del c.resp resp.error = None m.remove_handle(c) resp.finalize(c) goodlist.append(resp) for c, errno, errmsg in err_list: resp = c.resp del c.resp resp.error = (errno, errmsg) m.remove_handle(c) errlist.append(resp) if num_q == 0: break m.close() return goodlist, errlist
Example #12
Source File: utils.py From marathon-lb with Apache License 2.0 | 4 votes |
def __init__(self, url, auth, verify): self.url = url self.received_buffer = BytesIO() headers = ['Cache-Control: no-cache', 'Accept: text/event-stream'] self.curl = pycurl.Curl() self.curl.setopt(pycurl.URL, url) self.curl.setopt(pycurl.ENCODING, 'gzip') self.curl.setopt(pycurl.CONNECTTIMEOUT, 10) self.curl.setopt(pycurl.WRITEDATA, self.received_buffer) # Marathon >= 1.7.x returns 30x responses for /v2/events responses # when they're coming from a non-leader. So we follow redirects. self.curl.setopt(pycurl.FOLLOWLOCATION, True) self.curl.setopt(pycurl.MAXREDIRS, 1) self.curl.setopt(pycurl.UNRESTRICTED_AUTH, True) # The below settings are to prevent the connection from hanging if the # connection breaks silently. Since marathon-lb only listens, silent # connection failure results in marathon-lb waiting infinitely. # # Minimum bytes/second below which it is considered "low speed". So # "low speed" here refers to 0 bytes/second. self.curl.setopt(pycurl.LOW_SPEED_LIMIT, 1) # How long (in seconds) it's allowed to go below the speed limit # before it times out self.curl.setopt(pycurl.LOW_SPEED_TIME, 300) if auth and type(auth) is DCOSAuth: auth.refresh_auth_header() headers.append('Authorization: %s' % auth.auth_header) elif auth: self.curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) self.curl.setopt(pycurl.USERPWD, '%s:%s' % auth) if verify: self.curl.setopt(pycurl.CAINFO, verify) else: self.curl.setopt(pycurl.SSL_VERIFYHOST, 0) self.curl.setopt(pycurl.SSL_VERIFYPEER, 0) self.curl.setopt(pycurl.HTTPHEADER, headers) self.curlmulti = pycurl.CurlMulti() self.curlmulti.add_handle(self.curl) self.status_code = 0