Python six.moves.urllib.request.urlopen() Examples

def test_app_using_ssl(self):
        CONF.set_default('use_ssl', True)
        CONF.set_default("ssl_cert_file",
                         os.path.join(TEST_VAR_DIR, 'certificate.crt'))
        CONF.set_default("ssl_key_file",
                         os.path.join(TEST_VAR_DIR, 'privatekey.key'))

        greetings = 'Hello, World!!!'

        @webob.dec.wsgify
        def hello_world(req):
            return greetings

        server = wsgi.Server("test_app")
        server.start(hello_world, 0, host="127.0.0.1")

        response = urllibrequest.urlopen('https://127.0.0.1:%d/' % server.port)
        self.assertEqual(greetings, response.read())

        server.stop() 

def _readFromUrl(cls, url, writable):
        for attempt in retry_http():
            # We can only retry on errors that happen as responses to the request.
            # If we start getting file data, and the connection drops, we fail.
            # So we don't have to worry about writing the start of the file twice.
            with attempt:
                with closing(urlopen(url.geturl())) as readable:
                    # Make something to count the bytes we get
                    # We need to put the actual count in a container so our
                    # nested function can modify it without creating its own
                    # local with the same name.
                    size = [0]
                    def count(l):
                        size[0] += l
                    counter = WriteWatchingStream(writable)
                    counter.onWrite(count)
                    
                    # Do the download
                    shutil.copyfileobj(readable, counter)
                    return size[0] 

def _startMesos(self, numCores=None):
        if numCores is None:
            numCores = cpu_count()
        shutil.rmtree('/tmp/mesos', ignore_errors=True)
        self.master = self.MesosMasterThread(numCores)
        self.master.start()
        self.agent = self.MesosAgentThread(numCores)
        self.agent.start()
        
        # Wait for the master to come up.
        # Bad Things will happen if the master is not yet ready when Toil tries to use it.
        for attempt in retry(predicate=lambda e: True):
            with attempt:
                log.info('Checking if Mesos is ready...')
                with closing(urlopen('http://127.0.0.1:5050/version')) as content:
                    content.read()
        
        log.info('Mesos is ready! Running test.') 

def available_examples():
    """
    Return a list of available example data files from the nbodykit
    data repository on NERSC.

    Returns
    -------
    examples : list
        list of the available file names for download
    """
    # read the contents of the main data URL
    response = urlopen(data_url)
    contents = response.read().decode()

    # parse the available files
    parser = ListingParser(data_url)
    parser.feed(contents)

    # get relative paths and remove bad links
    available = [os.path.relpath(link, data_url) for link in parser.links]
    available = [link for link in available if not any(link.startswith(bad) for bad in ['.', '?'])]
    return sorted(available) 

def _getversions(self):
        """Fetch 'versions.csv'

        from versions_url config option and return the contained data as
        a dict with package name as a key.
        """
        versions_url = self.config_options.versions_url
        if not versions_url:
            fail_req_config_missing('versions_url')

        # return versions.csv as a dict with package name as a key
        vers = {}
        r = urlopen(versions_url)
        content = [x.decode('utf-8') for x in r.readlines()]
        # first line is headers
        for row in csv.reader(content[1:]):
            vers[row[0]] = row[1:]
        return vers 

def download_sifts_xml(pdb_id, outdir='', force_rerun=False):
    """Download the SIFTS file for a PDB ID.

    Args:
        pdb_id (str): PDB ID
        outdir (str): Output directory, current working directory if not specified.
        force_rerun (bool): If the file should be downloaded again even if it exists

    Returns:
        str: Path to downloaded file

    """
    baseURL = 'ftp://ftp.ebi.ac.uk/pub/databases/msd/sifts/xml/'
    filename = '{}.xml.gz'.format(pdb_id.lower())

    outfile = op.join(outdir, filename.split('.')[0] + '.sifts.xml')

    if ssbio.utils.force_rerun(flag=force_rerun, outfile=outfile):
        response = urlopen(baseURL + filename)
        with open(outfile, 'wb') as f:
            f.write(gzip.decompress(response.read()))

    return outfile 

def take_action(self, parsed_args):
        self.log.debug("take_action(%s)" % parsed_args)
        plan = parsed_args.plan
        outfile = parsed_args.output_file or '%s.tar.gz' % plan

        if os.path.exists(outfile) and not parsed_args.force_overwrite:
            raise exceptions.PlanExportError(
                "File '%s' already exists, not exporting." % outfile)

        print("Exporting plan %s..." % plan)

        tempurl = plan_management.export_deployment_plan(
            self.app.client_manager, plan
        )
        f = request.urlopen(tempurl)
        tarball_contents = f.read()
        f.close()

        with open(outfile, 'wb') as f:
            f.write(tarball_contents) 

def check_url(docker_ip, public_port):
    """Check if a service is reachable.

    Makes a simple GET request to '/' of the HTTP endpoint. Service is
    available if returned status code is < 500.
    """
    url = 'http://{}:{}'.format(docker_ip, public_port)
    try:
        r = urlopen(url)
        return r.code < 500
    except HTTPError as e:
        # If service returns e.g. a 404 it's ok
        return e.code < 500
    except Exception:
        # Possible service not yet started
        return False 

def _download_all_metadata(url):
    """
    Downloads the json file that contains all of the metadata for a specific file type (read:
    audio files, benchmark files, or trained models) that is on the EFZ server. This is retrieved
    from one of following three URLs (which are stored in nussl.constants):
    NUSSL_EFZ_AUDIO_METADATA_URL, NUSSL_EFZ_BENCHMARK_METADATA_URL, or NUSSL_EFZ_MODEL_METADATA_URL.

    Args:
        url (str):  URL for the EFZ server that has metadata. One of these three:
            NUSSL_EFZ_AUDIO_METADATA_URL, NUSSL_EFZ_BENCHMARK_METADATA_URL, or
            NUSSL_EFZ_MODEL_METADATA_URL.

    Returns:
        (list): List of dicts with metadata for the desired file type.

    """
    request = Request(url)

    # Make sure to get the newest data
    request.add_header('Pragma', 'no-cache')
    request.add_header('Cache-Control', 'max-age=0')
    try:
        return json.loads(urlopen(request).read())
    except:
        raise NoConnectivityError("Can't connect to internet") 

def raw_request(self, base_url, method_name, params, response_format):
        """
        Make a request to the Mixpanel API and return a raw urllib2/url.request file-like
        response object.
        """
        params['format'] = response_format
        # Getting rid of the None params
        params = self.check_params(params)
        url_without_params = '{base_url}/{version}/{method_name}/'.format(
            base_url=base_url,
            version=self.VERSION,
            method_name=method_name,
        )
        request_obj = self.client.auth.authenticate(url_without_params, params)
        effective_timeout = self.DEFAULT_TIMEOUT if self.client.timeout is None else self.client.timeout
        return url_request.urlopen(request_obj, timeout=effective_timeout) 

def _is_pingable(self, mgmt_ip='', retry=5, timeout=5, port=80, **kwargs):
        """Checks whether the server is reachable by using urllib.

        Waits for connectivity for `timeout` seconds,
        and if connection refused, it will retry `retry`
        times.
        :param mgmt_ip: IP to check
        :param retry: times to reconnect if connection refused
        :param timeout: seconds to wait for connection
        :param port: port number to check connectivity
        :return: bool - True or False depending on pingability.
        """
        url = 'http://' + mgmt_ip + ':' + str(port)
        if netaddr.valid_ipv6(mgmt_ip):
            url = 'http://[' + mgmt_ip + ']:' + str(port)

        for retry_index in range(int(retry)):
            try:
                urlreq.urlopen(url, timeout=timeout)
                return True
            except urlerr.URLError:
                LOG.warning('Unable to reach to the url %s', url)
        return 'failure' 

def test_app(self):
        self.skipTest("Not ready yet")
        greetings = 'Hello, World!!!'

        def hello_world(env, start_response):
            if env['PATH_INFO'] != '/':
                start_response('404 Not Found',
                               [('Content-Type', 'text/plain')])
                return ['Not Found\r\n']
            start_response('200 OK', [('Content-Type', 'text/plain')])
            return [greetings]

        server = wsgi.Server("test_app")
        server.start(hello_world, 0, host="127.0.0.1")

        response = urllibrequest.urlopen('http://127.0.0.1:%d/' % server.port)
        self.assertEqual(greetings, response.read())

        server.stop() 

def download_itis_database():
    print("Downloading ITIS data from: " + ITIS_URL)
    try:
        url = request.urlopen(ITIS_URL)
    except URLError:
        print("If you are operating behind a firewall, try setting the HTTP_PROXY/HTTPS_PROXY environment variables.")
        raise
    zipfile = ZipFile(BytesIO(url.read(int(url.headers['content-length']))))
    print("Download complete")
    named_temp_file = NamedTemporaryFile()
    itis_version = zipfile.filelist[0].filename.split('/')[0]
    db_file = None
    for f in zipfile.filelist:
        if f.filename.endswith('.sqlite'):
            db_file = f
            break
    with zipfile.open(db_file) as open_db_file:
        named_temp_file.write(open_db_file.read())
        named_temp_file.flush()
    return named_temp_file, itis_version 

def mocked_urlopen(url):
    if url.startswith('http://example.com'):
        fp = open('./dlrn/tests/samples/commits_remote.yaml', 'rb')
        return fp
    else:
        return urlopen(url) 

def validate(self, value):
        super(CaptchaField, self).validate(value)
        params = urlencode({
            'secret': settings.RECAPTCHA_SECRET_KEY,
            'response': smart_bytes(value),
            'remoteip': self.get_remoteip(),
        })
        url = 'https://www.google.com/recaptcha/api/siteverify?%s' % params
        if not loads(smart_text(urlopen(url).read())).get('success', False):
            raise ValidationError("Incorrect captcha solution provided.")
        return value 

def send(self, payload):
        path, content_type = self._get_path_content_type(payload)
        url = "http://{}:{}{}".format(self.address, self.port, path)

        req = Request(url, payload, {"Content-Type": content_type})
        response = urlopen(req)

        assert response.getcode() == 202 

def _body(self):
        return request.urlopen(self._url.geturl()) 

def _download(self, dstFile):
        """
        Download this resource from its URL to the given file object.

        :type dstFile: io.BytesIO|io.FileIO
        """
        for attempt in retry(predicate=lambda e: isinstance(e, HTTPError) and e.code == 400):
            with attempt:
                with closing(urlopen(self.url)) as content:
                    buf = content.read()
        contentHash = hashlib.md5(buf)
        assert contentHash.hexdigest() == self.contentHash
        dstFile.write(buf) 

def wait_api_port_ready(api_port, host='127.0.0.1'):
    """Wait until an http services becomes available

    :param api_port: api service port
    :type  api_port: integer

    :param host: host running the service (default: 127.0.0.1)
    :type host: string

    :return boolean
    """
    log = logging.getLogger(__name__ + ".wait_api_port_ready")
    urlopen_timeout = 1
    max_retries = 30
    count = 0
    while count < max_retries:
        time.sleep(1)
        count += 1
        try:
            request.urlopen(
                "http://%s:%s/" % (host, api_port), timeout=urlopen_timeout)
            return False
        except url_error.HTTPError as he:
            if he.code == 300:
                return True
            pass
        except url_error.URLError:
            pass
        except socket.timeout:
            log.warning(
                "Timeout at attempt {} of {} after {}s waiting for API port..."
                .format(count, max_retries, urlopen_timeout))
            pass
    raise RuntimeError(
        "wait_api_port_ready: Max retries {} reached".format(max_retries)) 

def urlretrieve(url, filename, reporthook=None, data=None):
        """Replacement for `urlretrive` for Python 2.

        Under Python 2, `urlretrieve` relies on `FancyURLopener` from legacy
        `urllib` module, known to have issues with proxy management.

        # Arguments
            url: url to retrieve.
            filename: where to store the retrieved data locally.
            reporthook: a hook function that will be called once
                on establishment of the network connection and once
                after each block read thereafter.
                The hook will be passed three arguments;
                a count of blocks transferred so far,
                a block size in bytes, and the total size of the file.
            data: `data` argument passed to `urlopen`.
        """

        def chunk_read(response, chunk_size=8192, reporthook=None):
            content_type = response.info().get('Content-Length')
            total_size = -1
            if content_type is not None:
                total_size = int(content_type.strip())
            count = 0
            while True:
                chunk = response.read(chunk_size)
                count += 1
                if reporthook is not None:
                    reporthook(count, chunk_size, total_size)
                if chunk:
                    yield chunk
                else:
                    break

        with closing(urlopen(url, data)) as response, open(filename, 'wb') as fd:
                for chunk in chunk_read(response, reporthook=reporthook):
                    fd.write(chunk) 

def urlretrieve(url, filename, reporthook=None, data=None):
        """Replacement for `urlretrive` for Python 2.

        Under Python 2, `urlretrieve` relies on `FancyURLopener` from legacy
        `urllib` module, known to have issues with proxy management.

        # Arguments
            url: url to retrieve.
            filename: where to store the retrieved data locally.
            reporthook: a hook function that will be called once
                on establishment of the network connection and once
                after each block read thereafter.
                The hook will be passed three arguments;
                a count of blocks transferred so far,
                a block size in bytes, and the total size of the file.
            data: `data` argument passed to `urlopen`.
        """

        def chunk_read(response, chunk_size=8192, reporthook=None):
            content_type = response.info().get('Content-Length')
            total_size = -1
            if content_type is not None:
                total_size = int(content_type.strip())
            count = 0
            while True:
                chunk = response.read(chunk_size)
                count += 1
                if reporthook is not None:
                    reporthook(count, chunk_size, total_size)
                if chunk:
                    yield chunk
                else:
                    break

        with closing(urlopen(url, data)) as response, open(filename, 'wb') as fd:
                for chunk in chunk_read(response, reporthook=reporthook):
                    fd.write(chunk) 

def urlretrieve(url, filename, reporthook=None, data=None):
        """Replacement for `urlretrive` for Python 2.

        Under Python 2, `urlretrieve` relies on `FancyURLopener` from legacy
        `urllib` module, known to have issues with proxy management.

        # Arguments
            url: url to retrieve.
            filename: where to store the retrieved data locally.
            reporthook: a hook function that will be called once
                on establishment of the network connection and once
                after each block read thereafter.
                The hook will be passed three arguments;
                a count of blocks transferred so far,
                a block size in bytes, and the total size of the file.
            data: `data` argument passed to `urlopen`.
        """

        def chunk_read(response, chunk_size=8192, reporthook=None):
            content_type = response.info().get('Content-Length')
            total_size = -1
            if content_type is not None:
                total_size = int(content_type.strip())
            count = 0
            while True:
                chunk = response.read(chunk_size)
                count += 1
                if reporthook is not None:
                    reporthook(count, chunk_size, total_size)
                if chunk:
                    yield chunk
                else:
                    break

        with closing(urlopen(url, data)) as response, open(filename, 'wb') as fd:
                for chunk in chunk_read(response, reporthook=reporthook):
                    fd.write(chunk) 

def urlretrieve(url, filename, reporthook=None, data=None):
        """Replacement for `urlretrive` for Python 2.

        Under Python 2, `urlretrieve` relies on `FancyURLopener` from legacy
        `urllib` module, known to have issues with proxy management.

        # Arguments
            url: url to retrieve.
            filename: where to store the retrieved data locally.
            reporthook: a hook function that will be called once
                on establishment of the network connection and once
                after each block read thereafter.
                The hook will be passed three arguments;
                a count of blocks transferred so far,
                a block size in bytes, and the total size of the file.
            data: `data` argument passed to `urlopen`.
        """

        def chunk_read(response, chunk_size=8192, reporthook=None):
            content_type = response.info().get('Content-Length')
            total_size = -1
            if content_type is not None:
                total_size = int(content_type.strip())
            count = 0
            while True:
                chunk = response.read(chunk_size)
                count += 1
                if reporthook is not None:
                    reporthook(count, chunk_size, total_size)
                if chunk:
                    yield chunk
                else:
                    break

        with closing(urlopen(url, data)) as response, open(filename, 'wb') as fd:
                for chunk in chunk_read(response, reporthook=reporthook):
                    fd.write(chunk) 

def open(self, filename=None, textmode=False):
        if filename is None:
            filename = self._base_uri
        else:
            if self._file_type == 's3':
                filename = urljoin(self._base_uri.replace(
                    's3://', 'http://'), filename.replace('\\', '/')).replace('http://', 's3://')
            elif self._file_type == 'http':
                filename = urljoin(self._base_uri, filename.replace('\\', '/'))
            else:
                filename = os.path.abspath(os.path.join(os.path.dirname(
                    self._base_uri.replace('\\', '/')), filename.replace('\\', '/')))
        f = None
        if self._file_type == 's3':
            uri_header, uri_body = filename.split('://', 1)
            us = uri_body.split('/')
            bucketname = us.pop(0)
            key = '/'.join(us)
            logger.info('Opening {}'.format(key))
            if textmode:
                f = StringIO(self.read_s3_object(key).decode('utf-8'))
            else:
                f = BytesIO(self.read_s3_object(key))
        elif self._file_type == 'http':
            f = request.urlopen(filename)
        else:
            if textmode:
                f = open(filename, 'rt')
            else:
                f = open(filename, 'rb')
        yield f
        f.close() 

def mocked_urlopen(url):
    if url.startswith('http://example.com'):
        fp = open('./dlrn/tests/samples/commits_remote.yaml', 'rb')
        return fp
    else:
        return urlopen(url) 

def download_and_resize_image(url, new_width=256, new_height=256,
                              display=False):
  _, filename = tempfile.mkstemp(suffix=".jpg")
  response = urlopen(url)
  image_data = response.read()
  image_data = BytesIO(image_data)
  pil_image = Image.open(image_data)
  pil_image = ImageOps.fit(pil_image, (new_width, new_height), Image.ANTIALIAS)
  pil_image_rgb = pil_image.convert("RGB")
  pil_image_rgb.save(filename, format="JPEG", quality=90)
  print("Image downloaded to %s." % filename)
  if display:
    display_image(pil_image)
  return filename 

def _fetch_reference_commit(self):
        """Fetch the commit.yaml file used for this run

        """
        versions_url = self.config_options.versions_url
        commit_url = re.sub('/versions.csv$', '/commit.yaml', versions_url)
        try:
            r = urlopen(commit_url)
            content = [x.decode('utf-8') for x in r.readlines()]
            return content
        except Exception:
            # We do not want to fail on this
            return None 

def video_path(tmpdir_factory):
    file = urlopen("http://storage.googleapis.com/cloud-samples-data/video/cat.mp4")
    path = tmpdir_factory.mktemp("video").join("file.mp4")
    with open(str(path), "wb") as f:
        f.write(file.read())

    return str(path) 

def spec_from_url(spec_url):
    content = request.urlopen(spec_url)
    return safe_load(content) 

def read_geonames_csv():
    print("Downloading geoname data from: " + GEONAMES_ZIP_URL)
    try:
        url = request.urlopen(GEONAMES_ZIP_URL)
    except URLError:
        print("If you are operating behind a firewall, try setting the HTTP_PROXY/HTTPS_PROXY environment variables.")
        raise
    zipfile = ZipFile(BytesIO(url.read()))
    print("Download complete")
    # Loading geonames data may cause errors without setting csv.field_size_limit:
    if sys.platform == "win32":
        max_c_long_on_windows = (2**32 / 2) - 1
        csv.field_size_limit(max_c_long_on_windows)
    else:
        csv.field_size_limit(sys.maxint if six.PY2 else six.MAXSIZE)
    with zipfile.open('allCountries.txt') as f:
        reader = unicodecsv.DictReader(f,
                                       fieldnames=[
                                           k for k, v in geonames_field_mappings],
                                       encoding='utf-8',
                                       delimiter='\t',
                                       quoting=csv.QUOTE_NONE)
        for d in reader:
            d['population'] = parse_number(d['population'], 0)
            d['latitude'] = parse_number(d['latitude'], 0)
            d['longitude'] = parse_number(d['longitude'], 0)
            if len(d['alternatenames']) > 0:
                d['alternatenames'] = d['alternatenames'].split(',')
            else:
                d['alternatenames'] = []
            yield d