Python urllib.urlopen() Examples

def send(self, note, number):
        conf = Configuration.conf()

        if not conf.get('sms_http_url'):
            raise ValueError(_("No SMS HTTP gateway defined"))

        params = urllib.urlencode({
            'username'  : conf['sms_http_user'],
            'password'  : conf['sms_http_password'],
            'text'      : note.body.encode('utf8'),
            'to'        : number
        })

        f = urllib.urlopen("%s?%s" % (conf['sms_http_url'], params),
                           context=_create_unverified_context())
        return f.read() 

def loadData():
    data_url = "data/multi-woz/data.json"
    dataset_url = "https://www.repository.cam.ac.uk/bitstream/handle/1810/280608/MULTIWOZ2.zip?sequence=3&isAllowed=y"
    if not os.path.exists("data"):
        os.makedirs("data")
        os.makedirs("data/multi-woz")

    if not os.path.exists(data_url):
        print("Downloading and unzipping the MultiWOZ dataset")
        resp = urllib.urlopen(dataset_url)
        zip_ref = ZipFile(BytesIO(resp.read()))
        zip_ref.extractall("data/multi-woz")
        zip_ref.close()
        shutil.copy('data/multi-woz/MULTIWOZ2 2/data.json', 'data/multi-woz/')
        shutil.copy('data/multi-woz/MULTIWOZ2 2/valListFile.json', 'data/multi-woz/')
        shutil.copy('data/multi-woz/MULTIWOZ2 2/testListFile.json', 'data/multi-woz/')
        shutil.copy('data/multi-woz/MULTIWOZ2 2/dialogue_acts.json', 'data/multi-woz/') 

def getPage( self, event ):
      """Parse URL, add addressing scheme and retrieve file"""

      # parse the URL      
      myURL = event.widget.get()
      components = urlparse.urlparse( myURL )
      self.contents.text_state = NORMAL

      # if addressing scheme not specified, use http
      if components[ 0 ] == "":
         myURL = "http://" + myURL

      # connect and retrieve the file
      try:
         tempFile = urllib.urlopen( myURL )
         self.contents.settext( tempFile.read() ) # show results 
         tempFile.close()
      except IOError:
         self.contents.settext( "Error finding file" )

      self.contents.text_state = DISABLED 

def Dispatch(self, ecb):
        print 'IIS dispatching "%s"' % (ecb.GetServerVariable("URL"),)
        url = ecb.GetServerVariable("URL")

        new_url = proxy + url
        print "Opening %s" % new_url
        fp = urllib.urlopen(new_url)
        headers = fp.info()
        ecb.SendResponseHeaders("200 OK", str(headers) + "\r\n", False)
        # now send the first chunk asynchronously
        ecb.ReqIOCompletion(io_callback, fp)
        chunk = fp.read(CHUNK_SIZE)
        if chunk:
            ecb.WriteClient(chunk, isapicon.HSE_IO_ASYNC)
            return isapicon.HSE_STATUS_PENDING
        # no data - just close things now.
        ecb.DoneWithSession()
        return isapicon.HSE_STATUS_SUCCESS

# The entry points for the ISAPI extension. 

def download_page(url, data=None):
    """Returns the response for the given url. The optional data argument is
    passed directly to urlopen.

    Args:
        url (str): The URL to read.
        data (Optional[any]): If given, a POST request will be made with
            :param:`data` as the POST body.

    Returns:
        str: The results of requesting the URL.
    """
    conn = urllib.urlopen(url, data)
    resp = conn.read()
    conn.close()
    return resp 

def rescan(self, resource):
        if os.path.isfile(resource):
            f = open(resource, 'rb')
            resource = hashlib.sha256(f.read()).hexdigest()
            f.close()
        url = self.api_url + "file/rescan"
        parameters = {"resource":  resource, "apikey": self.api_key }
        data = urllib.urlencode(parameters)
        req = urllib2.Request(url, data)
        try:
            response = urllib2.urlopen(req)
            xjson = response.read()
            response_code = json.loads(xjson).get('response_code')
            verbose_msg = json.loads(xjson).get('verbose_msg')
            if response_code == 1:
                print verbose_msg
                return xjson
            else:
                print verbose_msg
                
        except urllib2.HTTPError, e:
            self.handleHTTPErros(e.code) 

def get_remote_info_json(jfname):
   try:
      logging.info('loading example '+jfname)
      rfo = urllib.urlopen(jfname)
      di = json.loads(rfo.read())
      nat, glbs = 0, 0
      for k,v in di.items():
        if k != 'dimensions' or k != 'variables':
            glbs +=1 
      for k,v in di['variables'].items():
         for a in v: nat += 1  
      dims = [ l for k, v in di['dimensions'].items() for d, l in v.items() if d == 'length' ]
      return { 'num global attr' : glbs, 'num vars' : len(di['variables'].keys()), 'num dims' : \
               len(di['dimensions'].keys()), 'ave attrs per var' : nat / len(di['variables'].keys()), \
               'dims sizes' : dims }
   except Exception, e:
      logging.warn("WARN get_remote_info_json on %s : %s, update S3 bucket" % (jfname, str(e)))
      return {}

#--------------------------------------------------------------------------------- 

def getdomain(self, domain):
        url = self.api_url + "domain/report"
        parameters = {"domain": domain, "apikey": self.api_key}
        try:
            response = urllib.urlopen('%s?%s' % (url, urllib.urlencode(parameters))).read()
            xjson = json.loads(response)
            response_code = xjson['response_code']
            verbose_msg = xjson['verbose_msg']
            if response_code == 1:
                print verbose_msg
                return xjson
            else:
                print verbose_msg
                
        except urllib2.HTTPError, e:
            self.handleHTTPErros(e.code) 

def getip (self, ip):
        url = self.api_url + "ip-address/report"
        parameters = {"ip": ip, "apikey": self.api_key}
        try:
            response = urllib.urlopen('%s?%s' % (url, urllib.urlencode(parameters))).read()
            xjson = json.loads(response)
            response_code = xjson['response_code']
            verbose_msg = xjson['verbose_msg']
            if response_code == 1:
                print verbose_msg
                return xjson
            else:
                print verbose_msg
                
        except urllib2.HTTPError, e:
            self.handleHTTPErros(e.code) 

def geturl(self, resource):
        url = self.api_url + "url/report" 
        parameters = {"resource": resource, "apikey": self.api_key}
        data = urllib.urlencode(parameters)
        req = urllib2.Request(url, data)
        try:
            response = urllib2.urlopen(req)
            xjson = response.read()
            response_code = json.loads(xjson).get('response_code')
            verbose_msg = json.loads(xjson).get('verbose_msg')
            if response_code == 1:
                print verbose_msg
                return self.report(xjson)
            else:
                print verbose_msg
        
        except urllib2.HTTPError, e:
            self.handleHTTPErros(e.code) 

def getfile(self, file):
        if os.path.isfile(file):
            f = open(file, 'rb')
            file = hashlib.sha256(f.read()).hexdigest()
            f.close()
        url = self.api_url + "file/report"
        parameters = {"resource": file, "apikey": self.api_key}
        data = urllib.urlencode(parameters)
        req = urllib2.Request(url, data)
        try:
            response = urllib2.urlopen(req)
            xjson = response.read()
            response_code = json.loads(xjson).get('response_code')
            verbose_msg = json.loads(xjson).get('verbose_msg')
            if response_code == 1:
                print verbose_msg
                return self.report(xjson)
            else:
                print verbose_msg
                
        except urllib2.HTTPError, e:
            self.handleHTTPErros(e.code) 

def scanurl(self, link):
        url = self.api_url + "url/scan"
        parameters = {"url": link, "apikey": self.api_key}
        data = urllib.urlencode(parameters)
        req = urllib2.Request(url, data)
        try:
            response = urllib2.urlopen(req)
            xjson = response.read()
            response_code = json.loads(xjson).get('response_code')
            verbose_msg = json.loads(xjson).get('verbose_msg')
            if response_code == 1:
                print verbose_msg
                return xjson
            else:
                print verbose_msg
        
        except urllib2.HTTPError, e:
            self.handleHTTPErros(e.code) 

def send_sms_builtin(self, recipient, sender=None):
        """
        Sends SMS through built-in gateway
        """
        if not settings.SMS_HTTP_URL:
            raise ValueError(_('System is not configured for built-in SMS support.'))

        if sender is None:
            location = self.created_by.location
            sender = location.title

        data = urllib.urlencode({
            'username'  : settings.SMS_HTTP_USERNAME,
            'password'  : settings.SMS_HTTP_PASSWORD,
            'numberto'  : recipient.replace(' ', ''),
            'numberfrom': sender.encode(SMS_ENCODING),
            'message'   : self.body.encode(SMS_ENCODING),
        })

        from ssl import _create_unverified_context
        f = urllib.urlopen(settings.SMS_HTTP_URL, data, context=_create_unverified_context())
        return f.read() 

def test_distribute(self):
        real_urlopen = urllib.urlopen
        real_server_proxy = xmlrpclib.ServerProxy
        try:
            xmlrpclib.ServerProxy = ProxyStub(
                "distributedata.py", xmlrpclib.ServerProxy, False
            )
            urllib.urlopen = urlopenstub
            data = pypidata.get_data("distribute")
            rating = rate(data)

            self.assertEqual(
                rating,
                (
                    9,
                    [
                        "The classifiers should specify what minor versions of Python "
                        "you support as well as what major version.",
                        "You should have three or more owners of the project on PyPI.",
                    ],
                ),
            )
        finally:
            xmlrpclib.ServerProxy = real_server_proxy
            urllib.urlopen = real_urlopen 

def read_file_or_url(self, fname):
        # TODO: not working on localhost
        if isinstance(fname, file):
            result = open(fname, 'r')
        else:
            match = self.urlre.match(fname)
            if match:
                result = urllib.urlopen(match.group(1))
            else:
                fname = os.path.expanduser(fname)
                try:
                    result = open(os.path.expanduser(fname), 'r')
                except IOError:
                    result = open('%s.%s' % (os.path.expanduser(fname),
                                             self.defaultExtension), 'r')
        return result 

def run_simulation(self):
        global TARGET, ADDR, VULN_VAR, TIMEOUT, REQ_TOTAL,\
            METHOD, OTHER_VARIABLES
        tmp = OTHER_VARIABLES
        tmp[VULN_VAR] = self.genome
        try:
            if METHOD == 0:
                prep = urllib.urlencode(tmp)
                r = urllib.urlopen('http://%s/%s' % (TARGET, ADDR), data=prep, timeout=TIMEOUT)
            else:
                prep = urllib.urlencode(tmp)
                req = urllib.Request('http://%s/%s' % (TARGET, ADDR), data=prep)
                r = urllib.urlopen(req)
            REQ_TOTAL += 1
            self.m_text['text'] = r.get_data()
            self.m_text['url'] = r.get_full_url()
            self.m_text['status_code'] = r.getcode()
        except:
            pass
        return self.m_text 

def download_file(url, destination=''):
    if not destination:
        fd, destination = tempfile.mkstemp()
        os.remove(destination)
        os.close(fd)

    if not os.path.isfile(destination):
        ctx.logger.info('Downloading {0} to {1}...'.format(
            url, destination))
        try:
            final_url = urllib.urlopen(url).geturl()
            if final_url != url:
                ctx.logger.debug('Redirected to {0}'.format(final_url))
            f = urllib.URLopener()
            # TODO: try except with @retry
            f.retrieve(final_url, destination)
        except Exception:
            curl_download_with_retries(url, destination)
    else:
        ctx.logger.debug('File {0} already exists...'.format(destination))
    return destination 

def http_request(url,
                 data=None,
                 method='PUT',
                 headers=None,
                 timeout=None,
                 should_fail=False):
    headers = headers or {}
    request = urllib2.Request(url, data=data, headers=headers)
    request.get_method = lambda: method
    try:
        if timeout:
            return urllib2.urlopen(request, timeout=timeout)
        return urllib2.urlopen(request)
    except urllib2.URLError as e:
        if not should_fail:
            ctx.logger.error('Failed to {0} {1} (reason: {2})'.format(
                method, url, e.reason)) 

def hanlp_releases(cache=True):
    global HANLP_RELEASES
    if cache and HANLP_RELEASES:
        return HANLP_RELEASES
    # print('Request GitHub API')
    req = urllib.Request('http://nlp.hankcs.com/download.php?file=version')
    req.add_header('User-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36')
    if PY == 3:
        content = urllib.urlopen(req).read()
    else:
        content = urllib.urlopen(req).read()
    content = json.loads(content.decode())
    jar_version, jar_url, data_version, data_url = content
    meta = [(jar_version, jar_url, data_version, data_url)]
    HANLP_RELEASES = meta
    return meta 

def download_dataset(path, source='https://www.cs.toronto.edu/~kriz/'
                                  'cifar-10-python.tar.gz'):
    """
    Downloads and extracts the dataset, if needed.
    """
    files = ['data_batch_%d' % (i + 1) for i in range(5)] + ['test_batch']
    for fn in files:
        if not os.path.exists(os.path.join(path, 'cifar-10-batches-py', fn)):
            break  # at least one file is missing
    else:
        return  # dataset is already complete

    print("Downloading and extracting %s into %s..." % (source, path))
    if sys.version_info[0] == 2:
        from urllib import urlopen
    else:
        from urllib.request import urlopen
    import tarfile
    if not os.path.exists(path):
        os.makedirs(path)
    u = urlopen(source)
    with tarfile.open(fileobj=u, mode='r|gz') as f:
        f.extractall(path=path)
    u.close() 

def Dispatch(self, ecb):
        # Note that our ThreadPoolExtension base class will catch exceptions
        # in our Dispatch method, and write the traceback to the client.
        # That is perfect for this sample, so we don't catch our own.
        #print 'IIS dispatching "%s"' % (ecb.GetServerVariable("URL"),)
        url = ecb.GetServerVariable("URL")
        if url.startswith(virtualdir):
            new_url = proxy + url[len(virtualdir):]
            print "Opening", new_url
            fp = urllib.urlopen(new_url)
            headers = fp.info()
            ecb.SendResponseHeaders("200 OK", str(headers) + "\r\n", False)
            ecb.WriteClient(fp.read())
            ecb.DoneWithSession()
            print "Returned data from '%s'!" % (new_url,)
        else:
            # this should never happen - we should only see requests that
            # start with our virtual directory name.
            print "Not proxying '%s'" % (url,)


# The ISAPI filter. 

def test_crumb_capture(sentry_init, capture_events):
    sentry_init(integrations=[StdlibIntegration()])
    events = capture_events()

    url = "https://httpbin.org/status/200"
    response = urlopen(url)
    assert response.getcode() == 200
    capture_message("Testing!")

    (event,) = events
    (crumb,) = event["breadcrumbs"]
    assert crumb["type"] == "http"
    assert crumb["category"] == "httplib"
    assert crumb["data"] == {
        "url": url,
        "method": "GET",
        "status_code": 200,
        "reason": "OK",
    } 

def send(self):
        pwhash = md5(self.conf['sms_http_password']).hexdigest()

        params = {
            'username'  : self.conf['sms_http_user'],
            'password'  : pwhash,
            'message'   : self.body,
            'from'      : self.sender,
            'to'        : self.recipient,
        }

        if self.msg:
            dlruri = settings.SERVO_URL + '/api/messages/?id={0}'.format(self.msg.code)
            params['notify_url'] = dlruri

        params = urllib.urlencode(params)
        r = urllib.urlopen(self.URL, params, context=_create_unverified_context()).read()

        if 'ERROR:' in r:
            raise ValueError(self.ERRORS.get(r, _('Unknown error (%s)') % r)) 

def run(self, params={}):
        r = requests.session()
        url = self.connection.scraping_url

        pattern = re.compile(params.get("pattern"))
        req_params = {
            "api_dev_key": self.connection.dev_key,
            "limit": params.get("limit", "100"),
        }
        try:
            resp = r.post(url, data=req_params)  #urllib.urlopen(self.connection.scraping_url, urllib.urlencode(req_params), context=ctx)
        except Exception as e:
            self.logger.error("An error occurred ", e)
            raise
        data = resp.content.decode()
        data = json.loads(data)

        paste_list = []
        #pastes_raw = request.read()
        if resp.text.startswith("YOUR IP:"):
            self.logger.error('Scraping: IP not whitelisted')
        else:
            for item in data:
                scrape_url = item.get("scrape_url")
                resp = requests.get(scrape_url)
                response = resp.content.decode()
            if pattern.search(response) is not None:
                paste_list.append(item)
        results = dict()
        results["paste_list"] = paste_list
        self.logger.info(results)
        return results 

def test_bad_urllib_attribute(self):
        nodes = astroid.extract_node(
            """
        import urllib
        urllib.addbase #@
        urllib.splithost #@
        urllib.urlretrieve #@
        urllib.urlopen #@
        urllib.urlencode #@
        """
        )
        for node in nodes:
            message = testutils.Message("deprecated-urllib-function", node=node)
            with self.assertAddsMessages(message):
                self.checker.visit_attribute(node) 

def report(self, jsonx):
        if self._output == "json":
            return jsonx

        elif self._output == "html":
            jsonx = json.loads(jsonx)
            url = jsonx.get('permalink')
            print url
            html = urllib2.urlopen(url, timeout=3).read()
            return html.replace('<div class="frame" style="margin:20px 0">', '<a href="https://github.com/nu11p0inter/virustotal"> Exceuted by Tal Melamed [virustotal@appsec.it]</a> <div class="frame" style="margin:20px 0">')
        
        else: #print
            avlist = []
            jsonx = json.loads(jsonx)
            scan_date = jsonx.get('scan_date')
            total = jsonx.get('total')
            positive = jsonx.get('positives')
            print 'Scan date: ' + scan_date
            print 'Detection ratio: ' + str(positive) + "/" + str(total)
            scans = jsonx.get('scans')
            for av in scans.iterkeys():
                res = scans.get(av)
                if res.get('detected') == True:
                    avlist.append('+ ' + av + ':  ' + res.get('result'))
            if positive > 0:
                for res in avlist:
                    print res
                return avlist
            else:
                return 0

    # set a new api-key 

def get_content(url):
    try:  # Using Python3 urllib.
        with urllib.request.urlopen(url) as response:
            return response.read()  # Returns http.client.HTTPResponse.
    except AttributeError:  # Using Python3 urllib.
        return urllib.urlopen(url).read()  # Returns an instance. 

def test_bad_urllib_attribute(self):
        nodes = astroid.extract_node(
            """
        import urllib
        urllib.addbase #@
        urllib.splithost #@
        urllib.urlretrieve #@
        urllib.urlopen #@
        urllib.urlencode #@
        """
        )
        for node in nodes:
            message = testutils.Message("deprecated-urllib-function", node=node)
            with self.assertAddsMessages(message):
                self.checker.visit_attribute(node) 

def prepare_input_source(source, base = ""):
    """This function takes an InputSource and an optional base URL and
    returns a fully resolved InputSource object ready for reading."""

    if type(source) in _StringTypes:
        source = xmlreader.InputSource(source)
    elif hasattr(source, "read"):
        f = source
        source = xmlreader.InputSource()
        source.setByteStream(f)
        if hasattr(f, "name"):
            source.setSystemId(f.name)

    if source.getByteStream() is None:
        sysid = source.getSystemId()
        basehead = os.path.dirname(os.path.normpath(base))
        sysidfilename = os.path.join(basehead, sysid)
        if os.path.isfile(sysidfilename):
            source.setSystemId(sysidfilename)
            f = open(sysidfilename, "rb")
        else:
            source.setSystemId(urlparse.urljoin(base, sysid))
            f = urllib.urlopen(source.getSystemId())

        source.setByteStream(f)

    return source 

def fetch_public_key(repo):
    """Download RSA public key Travis will use for this repo.

    Travis API docs: http://docs.travis-ci.com/api/#repository-keys
    """
    keyurl = 'https://api.travis-ci.org/repos/{0}/key'.format(repo)
    data = json.loads(urlopen(keyurl).read().decode())
    if 'key' not in data:
        errmsg = "Could not find public key for repo: {}.\n".format(repo)
        errmsg += "Have you already added your GitHub repo to Travis?"
        raise ValueError(errmsg)
    return data['key']