Python urllib2.request() Examples

The following are 16 code examples of urllib2.request(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module urllib2 , or try the search function .
Example #1
Source File: client.py    From d6tpipe with MIT License 6 votes vote down vote up 
def _make_request(self, opener, request, timeout=None):
        """Make the API call and return the response. This is separated into
           it's own function, so we can mock it easily for testing.

        :param opener:
        :type opener:
        :param request: url payload to request
        :type request: urllib.Request object
        :param timeout: timeout value or None
        :type timeout: float
        :return: urllib response
        """
        timeout = timeout or self.timeout
        try:
            return opener.open(request, timeout=timeout)
        except HTTPError as err:
            exc = handle_error(err)
            return exc
Example #2
Source File: gen_rst.py    From sklearn-theano with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
def _get_data(url):
    """Helper function to get data over http or from a local file"""
    if url.startswith('http://'):
        # Try Python 2, use Python 3 on exception
        try:
            resp = urllib.urlopen(url)
            encoding = resp.headers.dict.get('content-encoding', 'plain')
        except AttributeError:
            resp = urllib.request.urlopen(url)
            encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            pass
        elif encoding == 'gzip':
            data = StringIO(data)
            data = gzip.GzipFile(fileobj=data).read()
        else:
            raise RuntimeError('unknown encoding')
    else:
        with open(url, 'r') as fid:
            data = fid.read()
        fid.close()

    return data
Example #3
Source File: gen_rst.py    From tick with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
def _get_data(url):
    """Helper function to get data over http or from a local file"""
    if url.startswith('http://'):
        # Try Python 2, use Python 3 on exception
        try:
            resp = urllib.urlopen(url)
            encoding = resp.headers.dict.get('content-encoding', 'plain')
        except AttributeError:
            resp = urllib.request.urlopen(url)
            encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            pass
        elif encoding == 'gzip':
            data = StringIO(data)
            data = gzip.GzipFile(fileobj=data).read()
        else:
            raise RuntimeError('unknown encoding')
    else:
        with open(url, 'r') as fid:
            data = fid.read()
        fid.close()

    return data
Example #4
Source File: docs_resolv.py    From spectrum with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
def _get_data(url):
    """Helper function to get data over http or from a local file"""
    if url.startswith('http://'):
        # Try Python 2, use Python 3 on exception
        try:
            resp = urllib.urlopen(url)
            encoding = resp.headers.dict.get('content-encoding', 'plain')
        except AttributeError:
            resp = urllib.request.urlopen(url)
            encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            pass
        elif encoding == 'gzip':
            data = StringIO(data)
            data = gzip.GzipFile(fileobj=data).read()
        else:
            raise RuntimeError('unknown encoding')
    else:
        with open(url, 'r') as fid:
            data = fid.read()

    return data
Example #5
Source File: gen_rst.py    From tensorlib with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
def _get_data(url):
    """Helper function to get data over http or from a local file"""
    if url.startswith('http://'):
        # Try Python 2, use Python 3 on exception
        try:
            resp = urllib.urlopen(url)
            encoding = resp.headers.dict.get('content-encoding', 'plain')
        except AttributeError:
            resp = urllib.request.urlopen(url)
            encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            pass
        elif encoding == 'gzip':
            data = StringIO(data)
            data = gzip.GzipFile(fileobj=data).read()
        else:
            raise RuntimeError('unknown encoding')
    else:
        with open(url, 'r') as fid:
            data = fid.read()
        fid.close()

    return data
Example #6
Source File: gen_rst.py    From polylearn with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
def _get_data(url):
    """Helper function to get data over http or from a local file"""
    if url.startswith('http://'):
        # Try Python 2, use Python 3 on exception
        try:
            resp = urllib.urlopen(url)
            encoding = resp.headers.dict.get('content-encoding', 'plain')
        except AttributeError:
            resp = urllib.request.urlopen(url)
            encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            pass
        elif encoding == 'gzip':
            data = StringIO(data)
            data = gzip.GzipFile(fileobj=data).read()
        else:
            raise RuntimeError('unknown encoding')
    else:
        with open(url, 'r') as fid:
            data = fid.read()
        fid.close()

    return data
Example #7
Source File: profile.py    From python-http-client with MIT License 6 votes vote down vote up 
def make_request(self,
                     method,
                     request_body=None,
                     query_params=None,
                     request_headers=None):
        method = method.upper()
        if request_headers:
            self._set_headers(request_headers)
            request_body = json.dumps(request_body) if request_body else None
            query_params = query_params if query_params else None
            opener = urllib.build_opener()
            request = urllib.Request(self._build_url(query_params),
                                     data=request_body)
            for key, value in self.request_headers.iteritems():
                request.add_header(key, value)
            request.get_method = lambda: method
            self._response = opener.open(request)
            self._set_response(self._response)
            self._reset()
Example #8
Source File: client.py    From d6tpipe with MIT License 5 votes vote down vote up 
def _update_headers(self, request_headers):
        """Update the headers for the request

        :param request_headers: headers to set for the API call
        :type request_headers: dictionary
        :return: dictionary
        """
        self.request_headers.update(request_headers)
Example #9
Source File: make_request.py    From HTTPLang with MIT License 5 votes vote down vote up 
def GET(path):
    url = getURL()
    opener = getOpener()
    request = urllib.Request(url)
    response = opener.open(request)
    setValues(response)
Example #10
Source File: make_request.py    From HTTPLang with MIT License 5 votes vote down vote up 
def POST(path):
    url = getURL()
    opener = getOpener()
    request = urllib.Request(url, GLOBALS['POSTDATA'])
    response = opener.open(request)
    setValues(response)
Example #11
Source File: __init__.py    From article-date-extractor with MIT License 5 votes vote down vote up 
def extractArticlePublishedDate(articleLink, html = None):

    print("Extracting date from " + articleLink)

    articleDate = None

    try:
        articleDate = _extractFromURL(articleLink)

        if html is None:
            request = urllib.Request(articleLink)
            # Using a browser user agent, decreases the change of sites blocking this request - just a suggestion
            # request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36')
            html = urllib.build_opener().open(request).read()

        parsedHTML = BeautifulSoup(html,"lxml")

        possibleDate = _extractFromLDJson(parsedHTML)
        if possibleDate is None:
            possibleDate = _extractFromMeta(parsedHTML)
        if possibleDate is None:
            possibleDate = _extractFromHTMLTag(parsedHTML)


        articleDate = possibleDate

    except Exception as e:
        print("Exception in extractArticlePublishedDate for " + articleLink)
        print(e.args)   

    return articleDate
Example #12
Source File: test_unit.py    From python-http-client with MIT License 5 votes vote down vote up 
def _make_request(self, opener, request, timeout=None):
        if 200 <= self.response_code < 299:  # if successful code
            return MockResponse(self.response_code)
        else:
            raise handle_error(MockException(self.response_code))
Example #13
Source File: test_unit.py    From python-http-client with MIT License 5 votes vote down vote up 
def test__urllib_headers(self, maker):
        self.client._update_headers({'X-test': 'Test'})
        self.client.get()
        request = maker.call_args[0][1]
        self.assertIn('X-test', request.headers)
Example #14
Source File: test_unit.py    From python-http-client with MIT License 5 votes vote down vote up 
def test__urllib_method(self, maker):
        self.client.delete()
        request = maker.call_args[0][1]
        self.assertEqual(request.get_method(), 'DELETE')
Example #15
Source File: client.py    From python-http-client with MIT License 5 votes vote down vote up 
def _update_headers(self, request_headers):
        """Update the headers for the request

        :param request_headers: headers to set for the API call
        :type request_headers: dictionary
        :return: dictionary
        """
        self.request_headers.update(request_headers)
Example #16
Source File: examples.py    From python-scripts with GNU General Public License v3.0 5 votes vote down vote up 
def download_file():
    big_file_url = "http://www.python.org/ftp/python/3.3.2/Python-3.3.2.tgz"

    # Download the file
    if sys.version_info[0] == 3:
        f = urllib.request.urlopen(big_file_url)
    else:
        f = urllib.urlopen(big_file_url)

    # Get the total length of the file
    scale = int(f.headers["content-length"])
    chunk_size = 500

    bar = CustomProgressBar(length=50,
                            left_limit='[',
                            right_limit=']',
                            head_repr=None,
                            empty_repr=' ',
                            filled_repr='|',
                            start=0,
                            scale_start=0,
                            scale_end=scale)

    print("Downloading a big txt file: ")

    print_flag = 0
    # Load all the data chunk by chunk
    while not bar.completed():
        f.read(chunk_size)
        bar.increase(chunk_size)

        # Don't print always
        if print_flag == 100:
            bar.show_progress_bar()
            print_flag = 0
        else:
            print_flag += 1

    bar.show_progress_bar()
    print("")
    print("Finished :)")