Python urllib2.OpenerDirector() Examples

The following are 14 code examples of urllib2.OpenerDirector(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module urllib2 , or try the search function .
Example #1
Source File: parseopener.py    From vulscan with MIT License 5 votes vote down vote up 
def openerHeaders(op):
    headers = {}
    try:
        assert isinstance(op, urllib2.OpenerDirector)
        _ = op.addheaders
        for pair in _:
            # pair_copy = [part for part in pair]
            headers.update({pair[0]: pair[1]})
    except:
        errMsg = 'unable to fetch headers from given opener'
        logger.log(CUSTOM_LOGGING.ERROR, errMsg)
    return headers
Example #2
Source File: appengine_rpc.py    From browserscope with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector for making HTTP requests.

    Returns:
      A urllib2.OpenerDirector object.
    """
    raise NotImplementedError
Example #3
Source File: appengine_rpc.py    From browserscope with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector that supports cookies and ignores redirects.

    Returns:
      A urllib2.OpenerDirector object.
    """
    opener = urllib2.OpenerDirector()
    opener.add_handler(fancy_urllib.FancyProxyHandler())
    opener.add_handler(urllib2.UnknownHandler())
    opener.add_handler(urllib2.HTTPHandler())
    opener.add_handler(urllib2.HTTPDefaultErrorHandler())
    opener.add_handler(fancy_urllib.FancyHTTPSHandler())
    opener.add_handler(urllib2.HTTPErrorProcessor())
    opener.add_handler(ContentEncodingHandler())

    if self.save_cookies:
      self.cookie_jar.filename = os.path.expanduser(
          HttpRpcServer.DEFAULT_COOKIE_FILE_PATH)

      if os.path.exists(self.cookie_jar.filename):
        try:
          self.cookie_jar.load()
          self.authenticated = True
          logger.debug("Loaded authentication cookies from %s",
                       self.cookie_jar.filename)
        except (OSError, IOError, cookielib.LoadError), e:

          logger.debug("Could not load authentication cookies; %s: %s",
                       e.__class__.__name__, e)
          self.cookie_jar.filename = None
      else:


        try:
          fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600)
          os.close(fd)
        except (OSError, IOError), e:

          logger.debug("Could not create authentication cookies file; %s: %s",
                       e.__class__.__name__, e)
          self.cookie_jar.filename = None
Example #4
Source File: upload.py    From training_results_v0.5 with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector for making HTTP requests.

    Returns:
      A urllib2.OpenerDirector object.
    """
    raise NotImplementedError()
Example #5
Source File: upload.py    From training_results_v0.5 with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector that supports cookies and ignores redirects.

    Returns:
      A urllib2.OpenerDirector object.
    """
    opener = urllib2.OpenerDirector()
    opener.add_handler(urllib2.ProxyHandler())
    opener.add_handler(urllib2.UnknownHandler())
    opener.add_handler(urllib2.HTTPHandler())
    opener.add_handler(urllib2.HTTPDefaultErrorHandler())
    opener.add_handler(urllib2.HTTPSHandler())
    opener.add_handler(urllib2.HTTPErrorProcessor())
    if self.save_cookies:
      self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
      self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
      if os.path.exists(self.cookie_file):
        try:
          self.cookie_jar.load()
          self.authenticated = True
          StatusUpdate("Loaded authentication cookies from %s" %
                       self.cookie_file)
        except (cookielib.LoadError, IOError):
          # Failed to load cookies - just ignore them.
          pass
      else:
        # Create an empty cookie file with mode 600
        fd = os.open(self.cookie_file, os.O_CREAT, 0600)
        os.close(fd)
      # Always chmod the cookie file
      os.chmod(self.cookie_file, 0600)
    else:
      # Don't save cookies across runs of update.py.
      self.cookie_jar = cookielib.CookieJar()
    opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
    return opener
Example #6
Source File: upload.py    From training_results_v0.5 with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector for making HTTP requests.

    Returns:
      A urllib2.OpenerDirector object.
    """
    raise NotImplementedError()
Example #7
Source File: upload.py    From training_results_v0.5 with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector that supports cookies and ignores redirects.

    Returns:
      A urllib2.OpenerDirector object.
    """
    opener = urllib2.OpenerDirector()
    opener.add_handler(urllib2.ProxyHandler())
    opener.add_handler(urllib2.UnknownHandler())
    opener.add_handler(urllib2.HTTPHandler())
    opener.add_handler(urllib2.HTTPDefaultErrorHandler())
    opener.add_handler(urllib2.HTTPSHandler())
    opener.add_handler(urllib2.HTTPErrorProcessor())
    if self.save_cookies:
      self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
      self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
      if os.path.exists(self.cookie_file):
        try:
          self.cookie_jar.load()
          self.authenticated = True
          StatusUpdate("Loaded authentication cookies from %s" %
                       self.cookie_file)
        except (cookielib.LoadError, IOError):
          # Failed to load cookies - just ignore them.
          pass
      else:
        # Create an empty cookie file with mode 600
        fd = os.open(self.cookie_file, os.O_CREAT, 0600)
        os.close(fd)
      # Always chmod the cookie file
      os.chmod(self.cookie_file, 0600)
    else:
      # Don't save cookies across runs of update.py.
      self.cookie_jar = cookielib.CookieJar()
    opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
    return opener
Example #8
Source File: test_basefs.py    From familysearch-python-sdk-opensource with BSD 2-Clause "Simplified" License 5 votes vote down vote up 
def test_base_fs_creation(self):
        fs = familysearch.FamilySearch(self.agent, self.devkey)
        self.assertTrue(fs.base == 'https://sandbox.familysearch.org')
        print("Base is correct.")
        self.assertTrue(fs.key == self.devkey)
        print("Key is correct.")
        self.assertTrue(isinstance(fs.opener, request.OpenerDirector))
        print("HTTP opener works.")
        self.assertTrue(fs.access_token == None)
        print("Access token works.")
Example #9
Source File: appengine_rpc.py    From python-compat-runtime with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector for making HTTP requests.

    Returns:
      A urllib2.OpenerDirector object.
    """
    raise NotImplementedError
Example #10
Source File: appengine_rpc.py    From python-compat-runtime with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector that supports cookies and ignores redirects.

    Returns:
      A urllib2.OpenerDirector object.
    """
    opener = urllib2.OpenerDirector()
    opener.add_handler(fancy_urllib.FancyProxyHandler())
    opener.add_handler(urllib2.UnknownHandler())
    opener.add_handler(urllib2.HTTPHandler())
    opener.add_handler(urllib2.HTTPDefaultErrorHandler())
    opener.add_handler(fancy_urllib.FancyHTTPSHandler())
    opener.add_handler(urllib2.HTTPErrorProcessor())
    opener.add_handler(ContentEncodingHandler())

    if self.save_cookies:
      self.cookie_jar.filename = os.path.expanduser(
          HttpRpcServer.DEFAULT_COOKIE_FILE_PATH)

      if os.path.exists(self.cookie_jar.filename):
        try:
          self.cookie_jar.load()
          self.authenticated = True
          logger.debug("Loaded authentication cookies from %s",
                       self.cookie_jar.filename)
        except (OSError, IOError, cookielib.LoadError), e:

          logger.debug("Could not load authentication cookies; %s: %s",
                       e.__class__.__name__, e)
          self.cookie_jar.filename = None
      else:


        try:
          fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600)
          os.close(fd)
        except (OSError, IOError), e:

          logger.debug("Could not create authentication cookies file; %s: %s",
                       e.__class__.__name__, e)
          self.cookie_jar.filename = None
Example #11
Source File: throttle.py    From python-compat-runtime with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector that supports cookies and ignores redirects.

    Returns:
      A urllib2.OpenerDirector object.
    """
    opener = appengine_rpc.HttpRpcServer._GetOpener(self)
    opener.add_handler(ThrottleHandler(self.throttle))

    return opener
Example #12
Source File: upload.py    From personfinder with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector for making HTTP requests.

    Returns:
      A urllib2.OpenerDirector object.
    """
    raise NotImplementedError()
Example #13
Source File: upload.py    From personfinder with Apache License 2.0 5 votes vote down vote up 
def _GetOpener(self):
    """Returns an OpenerDirector that supports cookies and ignores redirects.

    Returns:
      A urllib2.OpenerDirector object.
    """
    opener = urllib2.OpenerDirector()
    opener.add_handler(urllib2.ProxyHandler())
    opener.add_handler(urllib2.UnknownHandler())
    opener.add_handler(urllib2.HTTPHandler())
    opener.add_handler(urllib2.HTTPDefaultErrorHandler())
    opener.add_handler(urllib2.HTTPSHandler())
    opener.add_handler(urllib2.HTTPErrorProcessor())
    if self.save_cookies:
      self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
      self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
      if os.path.exists(self.cookie_file):
        try:
          self.cookie_jar.load()
          self.authenticated = True
          StatusUpdate("Loaded authentication cookies from %s" %
                       self.cookie_file)
        except (cookielib.LoadError, IOError):
          # Failed to load cookies - just ignore them.
          pass
      else:
        # Create an empty cookie file with mode 600
        fd = os.open(self.cookie_file, os.O_CREAT, 0600)
        os.close(fd)
      # Always chmod the cookie file
      os.chmod(self.cookie_file, 0600)
    else:
      # Don't save cookies across runs of update.py.
      self.cookie_jar = cookielib.CookieJar()
    opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
    return opener
Example #14
Source File: lookasidetest.py    From conary with Apache License 2.0 4 votes vote down vote up 
def testAuth(self):
        # CNY-981
        try:
            contentServer = rephelp.HTTPServerController(authRequester())
            baseUrl = contentServer.url()

            # test with user:pass
            contentURL = 'http://user:pass@%s' % httplib.urlsplit(baseUrl)[1]
            name = 'foo.tar.gz'
            url = contentURL + '/' + name
            cached = lookaside.fetchURL(self.cfg, url, name)
            f = open(cached, 'r')
            self.assertEqual(f.read(), 'Hello, world!\n')

            # test with no password given
            contentURL = 'http://user@%s' % httplib.urlsplit(baseUrl)[1]
            name = 'foo2.tar.gz'
            url = contentURL + '/' + name
            cached = lookaside.fetchURL(self.cfg, url, name)
            f = open(cached, 'r')
            self.assertEqual(f.read(), 'Hello, world 2!\n')

            # test with no password at all
            name = 'foo3.tar.gz'
            url = baseUrl + '/' + name
            cached = self.logCheck(lookaside.fetchURL, (self.cfg, url, name),
                                   ['error: error downloading http://localhost:[0-9]*//foo3.tar.gz: HTTP Error 401: Unauthorized'],
                                   regExp=True)
            self.assertEqual(cached, None)

            # test ftp with user:pass
            def fakeOpen(od, req, *args, **kw):
                self.req = req
                import StringIO
                s = 'baz file contents'
                r = StringIO.StringIO(s)
                r.headers = {'contentLength': len(s)}
                return r

            import urllib2
            self.mock(urllib2.OpenerDirector, 'open', fakeOpen)
            url = 'ftp://user:pass@foo.com/bar/baz.tgz'
            name = 'baz.tgz'
            cached = lookaside.fetchURL(self.cfg, url, name)
            self.assertEqual(url, self.req.get_full_url())
            self.assertEqual(open(cached).read(), 'baz file contents')

        finally:
            contentServer.kill()