Python boto.utils() Examples

def add_auth(self, req, **kwargs):
        req.params['AWSAccessKeyId'] = self._provider.access_key
        req.params['SignatureVersion'] = self.SignatureVersion
        req.params['Timestamp'] = boto.utils.get_ts()
        qs, signature = self._calc_signature(req.params, req.method,
                                             req.auth_path, req.host)
        boto.log.debug('query_string: %s Signature: %s' % (qs, signature))
        if req.method == 'POST':
            req.headers['Content-Length'] = str(len(req.body))
            req.headers['Content-Type'] = req.headers.get('Content-Type',
                                                          'text/plain')
        else:
            req.body = ''
        # if this is a retried req, the qs from the previous try will
        # already be there, we need to get rid of that and rebuild it
        req.path = req.path.split('?')[0]
        req.path = (req.path + '?' + qs +
                    '&Signature=' + urllib.parse.quote_plus(signature)) 

def _calc_signature(self, params, verb, path, server_name):
        boto.log.debug('using _calc_signature_2')
        string_to_sign = '%s\n%s\n%s\n' % (verb, server_name.lower(), path)
        if self._hmac_256:
            hmac = self._hmac_256.copy()
            params['SignatureMethod'] = 'HmacSHA256'
        else:
            hmac = self._hmac.copy()
            params['SignatureMethod'] = 'HmacSHA1'
        keys = params.keys()
        keys.sort()
        pairs = []
        for key in keys:
            val = boto.utils.get_utf8_value(params[key])
            pairs.append(urllib.quote(key, safe='') + '=' +
                         urllib.quote(val, safe='-_~'))
        qs = '&'.join(pairs)
        boto.log.debug('query string: %s' % qs)
        string_to_sign += qs
        boto.log.debug('string_to_sign: %s' % string_to_sign)
        hmac.update(string_to_sign)
        b64 = base64.b64encode(hmac.digest())
        boto.log.debug('len(b64)=%d' % len(b64))
        boto.log.debug('base64 encoded digest: %s' % b64)
        return (qs, b64) 

def _calc_signature(self, params, verb, path, server_name):
        boto.log.debug('using _calc_signature_2')
        string_to_sign = '%s\n%s\n%s\n' % (verb, server_name.lower(), path)
        hmac = self._get_hmac()
        params['SignatureMethod'] = self.algorithm()
        if self._provider.security_token:
            params['SecurityToken'] = self._provider.security_token
        keys = sorted(params.keys())
        pairs = []
        for key in keys:
            val = boto.utils.get_utf8_value(params[key])
            pairs.append(urllib.parse.quote(key, safe='') + '=' +
                         urllib.parse.quote(val, safe='-_~'))
        qs = '&'.join(pairs)
        boto.log.debug('query string: %s' % qs)
        string_to_sign += qs
        boto.log.debug('string_to_sign: %s' % string_to_sign)
        hmac.update(string_to_sign.encode('utf-8'))
        b64 = base64.b64encode(hmac.digest())
        boto.log.debug('len(b64)=%d' % len(b64))
        boto.log.debug('base64 encoded digest: %s' % b64)
        return (qs, b64) 

def add_auth(self, http_request, **kwargs):
        headers = http_request.headers
        params = http_request.params
        params['AWSAccessKeyId'] = self._provider.access_key
        params['SignatureVersion'] = self.SignatureVersion
        params['Timestamp'] = boto.utils.get_ts()
        qs, signature = self._calc_signature(
            http_request.params, http_request.method,
            http_request.auth_path, http_request.host)
        boto.log.debug('query_string: %s Signature: %s' % (qs, signature))
        if http_request.method == 'POST':
            headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
            http_request.body = qs + '&Signature=' + urllib.quote(signature)
            http_request.headers['Content-Length'] = str(len(http_request.body))
        else:
            http_request.body = ''
            # if this is a retried request, the qs from the previous try will
            # already be there, we need to get rid of that and rebuild it
            http_request.path = http_request.path.split('?')[0]
            http_request.path = (http_request.path + '?' + qs + '&Signature=' + urllib.quote(signature)) 

def configure_lifecycle(self, lifecycle_config, headers=None):
        """
        Configure lifecycle for this bucket.

        :type lifecycle_config: :class:`boto.s3.lifecycle.Lifecycle`
        :param lifecycle_config: The lifecycle configuration you want
            to configure for this bucket.
        """
        xml = lifecycle_config.to_xml()
        #xml = xml.encode('utf-8')
        fp = StringIO(xml)
        md5 = boto.utils.compute_md5(fp)
        if headers is None:
            headers = {}
        headers['Content-MD5'] = md5[1]
        headers['Content-Type'] = 'text/xml'
        response = self.connection.make_request('PUT', self.name,
                                                data=fp.getvalue(),
                                                query_args='lifecycle',
                                                headers=headers)
        body = response.read()
        if response.status == 200:
            return True
        else:
            raise self.connection.provider.storage_response_error(
                response.status, response.reason, body) 

def set_xml_tags(self, tag_str, headers=None, query_args='tagging'):
        if headers is None:
            headers = {}
        md5 = boto.utils.compute_md5(StringIO(tag_str))
        headers['Content-MD5'] = md5[1]
        headers['Content-Type'] = 'text/xml'
        if not isinstance(tag_str, bytes):
            tag_str = tag_str.encode('utf-8')
        response = self.connection.make_request('PUT', self.name,
                                                data=tag_str,
                                                query_args=query_args,
                                                headers=headers)
        body = response.read()
        if response.status != 204:
            raise self.connection.provider.storage_response_error(
                response.status, response.reason, body)
        return True 

def _populate_keys_from_metadata_server(self):
        # get_instance_metadata is imported here because of a circular
        # dependency.
        boto.log.debug("Retrieving credentials from metadata server.")
        from boto.utils import get_instance_metadata
        timeout = config.getfloat('Boto', 'metadata_service_timeout', 1.0)
        attempts = config.getint('Boto', 'metadata_service_num_attempts', 1)
        # The num_retries arg is actually the total number of attempts made,
        # so the config options is named *_num_attempts to make this more
        # clear to users.
        metadata = get_instance_metadata(
            timeout=timeout, num_retries=attempts,
            data='meta-data/iam/security-credentials/')
        if metadata:
            creds = self._get_credentials_from_metadata(metadata)
            self._access_key = creds[0]
            self._secret_key = creds[1]
            self._security_token = creds[2]
            expires_at = creds[3]
            # I'm assuming there's only one role on the instance profile.
            self._credential_expiry_time = datetime.strptime(
                expires_at, "%Y-%m-%dT%H:%M:%SZ")
            boto.log.debug("Retrieved credentials will expire in %s at: %s",
                           self._credential_expiry_time - datetime.now(),
                           expires_at) 

def _getCurrentAWSZone(spotBid=None, nodeType=None, ctx=None):
    zone = None
    try:
        import boto
        from boto.utils import get_instance_metadata
    except ImportError:
        pass
    else:
        zone = os.environ.get('TOIL_AWS_ZONE', None)
        if not zone and runningOnEC2():
            try:
                zone = get_instance_metadata()['placement']['availability-zone']
            except KeyError:
                pass
        if not zone and spotBid:
            # if spot bid is present, all the other parameters must be as well
            assert bool(spotBid) == bool(nodeType) == bool(ctx)
            # if the zone is unset and we are using the spot market, optimize our
            # choice based on the spot history
            return optimize_spot_bid(ctx=ctx, instance_type=nodeType, spot_bid=float(spotBid))
        if not zone:
            zone = boto.config.get('Boto', 'ec2_region_name')
            if zone is not None:
                zone += 'a'  # derive an availability zone in the region
    return zone 

def __init__(self, verbose_name=None, name=None, default='', required=False,
                 validator=None, choices=None, unique=False, hashfunc=None):

        """
           The hashfunc parameter overrides the default hashfunc in boto.utils.Password.

           The remaining parameters are passed through to StringProperty.__init__"""


        StringProperty.__init__(self, verbose_name, name, default, required, validator, choices, unique)
        self.hashfunc=hashfunc 

def fetch_s3_file(self, s3_file):
        try:
            from boto.utils import fetch_file
            f = fetch_file(s3_file)
            path = os.path.join(self.working_dir, s3_file.split("/")[-1])
            open(path, "w").write(f.read())
        except:
            boto.log.exception('Problem Retrieving file: %s' % s3_file)
            path = None
        return path 

def _get_key_internal(self, key_name, headers, query_args_l):
        query_args = '&'.join(query_args_l) or None
        response = self.connection.make_request('HEAD', self.name, key_name,
                                                headers=headers,
                                                query_args=query_args)
        response.read()
        # Allow any success status (2xx) - for example this lets us
        # support Range gets, which return status 206:
        if response.status / 100 == 2:
            k = self.key_class(self)
            provider = self.connection.provider
            k.metadata = boto.utils.get_aws_metadata(response.msg, provider)
            for field in Key.base_fields:
                k.__dict__[field.lower().replace('-', '_')] = \
                    response.getheader(field)
            # the following machinations are a workaround to the fact that
            # apache/fastcgi omits the content-length header on HEAD
            # requests when the content-length is zero.
            # See http://goo.gl/0Tdax for more details.
            clen = response.getheader('content-length')
            if clen:
                k.size = int(response.getheader('content-length'))
            else:
                k.size = 0
            k.name = key_name
            k.handle_version_headers(response)
            k.handle_encryption_headers(response)
            k.handle_restore_headers(response)
            k.handle_storage_class_header(response)
            k.handle_addl_headers(response.getheaders())
            return k, response
        else:
            if response.status == 404:
                return None, response
            else:
                raise self.connection.provider.storage_response_error(
                    response.status, response.reason, '') 

def get_metadata():
    metadata = boto.utils.get_instance_metadata(timeout=1, num_retries=2)
    if not metadata:
        raise ValueError('Cannot obtain EC2 metadata.')
    return metadata 

def describe_jobflows(self, states=None, jobflow_ids=None,
                           created_after=None, created_before=None):
        """
        Retrieve all the Elastic MapReduce job flows on your account

        :type states: list
        :param states: A list of strings with job flow states wanted

        :type jobflow_ids: list
        :param jobflow_ids: A list of job flow IDs
        :type created_after: datetime
        :param created_after: Bound on job flow creation time

        :type created_before: datetime
        :param created_before: Bound on job flow creation time
        """
        params = {}

        if states:
            self.build_list_params(params, states, 'JobFlowStates.member')
        if jobflow_ids:
            self.build_list_params(params, jobflow_ids, 'JobFlowIds.member')
        if created_after:
            params['CreatedAfter'] = created_after.strftime(
                boto.utils.ISO8601)
        if created_before:
            params['CreatedBefore'] = created_before.strftime(
                boto.utils.ISO8601)

        return self.get_list('DescribeJobFlows', params, [('member', JobFlow)]) 

def add_auth(self, http_request, **kwargs):
        headers = http_request.headers
        method = http_request.method
        auth_path = http_request.auth_path
        if not headers.has_key('Date'):
            headers['Date'] = formatdate(usegmt=True)

        c_string = boto.utils.canonical_string(method, auth_path, headers,
                                               None, self._provider)
        b64_hmac = self.sign_string(c_string)
        auth_hdr = self._provider.auth_header
        headers['Authorization'] = ("%s %s:%s" %
                                    (auth_hdr,
                                     self._provider.access_key, b64_hmac)) 

def _calc_signature(self, params, *args):
        boto.log.debug('using _calc_signature_1')
        hmac = self._hmac.copy()
        keys = params.keys()
        keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower()))
        pairs = []
        for key in keys:
            hmac.update(key)
            val = boto.utils.get_utf8_value(params[key])
            hmac.update(val)
            pairs.append(key + '=' + urllib.quote(val))
        qs = '&'.join(pairs)
        return (qs, base64.b64encode(hmac.digest())) 

def get_utf8_value(self, value):
        return boto.utils.get_utf8_value(value) 

def notify(self, subject, body=''):
        boto.utils.notify(subject, body) 

def fetch_s3_file(self, s3_file):
        try:
            from boto.utils import fetch_file
            f = fetch_file(s3_file)
            path = os.path.join(self.working_dir, s3_file.split("/")[-1])
            open(path, "w").write(f.read())
        except:
            boto.log.exception('Problem Retrieving file: %s' % s3_file)
            path = None
        return path 

def get_utf8_value(self, value):
        return boto.utils.get_utf8_value(value) 

def parseS3Timestamp(timestamp):
    diff_from_epoch = boto.utils.parse_ts(timestamp) - datetime.datetime.utcfromtimestamp(0)
    return diff_from_epoch.total_seconds() 

def _check_token_cache(self, token_key, duration=None, window_seconds=60):
        token = _session_token_cache.get(token_key, None)
        if token:
            now = datetime.datetime.utcnow()
            expires = boto.utils.parse_ts(token.expiration)
            delta = expires - now
            if delta < datetime.timedelta(seconds=window_seconds):
                msg = 'Cached session token %s is expired' % token_key
                boto.log.debug(msg)
                token = None
        return token 

def __init__(self, verbose_name=None, name=None, default='', required=False,
                 validator=None, choices=None, unique=False, hashfunc=None):

        """
           The hashfunc parameter overrides the default hashfunc in boto.utils.Password.

           The remaining parameters are passed through to StringProperty.__init__"""

        super(PasswordProperty, self).__init__(verbose_name, name, default, required,
                                validator, choices, unique)
        self.hashfunc = hashfunc 

def describe_jobflows(self, states=None, jobflow_ids=None,
                           created_after=None, created_before=None):
        """
        This method is deprecated. We recommend you use list_clusters,
        describe_cluster, list_steps, list_instance_groups and
        list_bootstrap_actions instead.

        Retrieve all the Elastic MapReduce job flows on your account

        :type states: list
        :param states: A list of strings with job flow states wanted

        :type jobflow_ids: list
        :param jobflow_ids: A list of job flow IDs
        :type created_after: datetime
        :param created_after: Bound on job flow creation time

        :type created_before: datetime
        :param created_before: Bound on job flow creation time
        """
        params = {}

        if states:
            self.build_list_params(params, states, 'JobFlowStates.member')
        if jobflow_ids:
            self.build_list_params(params, jobflow_ids, 'JobFlowIds.member')
        if created_after:
            params['CreatedAfter'] = created_after.strftime(
                boto.utils.ISO8601)
        if created_before:
            params['CreatedBefore'] = created_before.strftime(
                boto.utils.ISO8601)

        return self.get_list('DescribeJobFlows', params, [('member', JobFlow)]) 

def list_clusters(self, created_after=None, created_before=None,
                      cluster_states=None, marker=None):
        """
        List Elastic MapReduce clusters with optional filtering

        :type created_after: datetime
        :param created_after: Bound on cluster creation time
        :type created_before: datetime
        :param created_before: Bound on cluster creation time
        :type cluster_states: list
        :param cluster_states: Bound on cluster states
        :type marker: str
        :param marker: Pagination marker
        """
        params = {}
        if created_after:
            params['CreatedAfter'] = created_after.strftime(
                boto.utils.ISO8601)
        if created_before:
            params['CreatedBefore'] = created_before.strftime(
                boto.utils.ISO8601)
        if marker:
            params['Marker'] = marker

        if cluster_states:
            self.build_list_params(params, cluster_states, 'ClusterStates.member')

        return self.get_object('ListClusters', params, ClusterSummaryList) 

def query_string(self, http_request):
        parameter_names = sorted(http_request.params.keys())
        pairs = []
        for pname in parameter_names:
            pval = boto.utils.get_utf8_value(http_request.params[pname])
            pairs.append(urllib.parse.quote(pname, safe='') + '=' +
                         urllib.parse.quote(pval, safe='-_~'))
        return '&'.join(pairs) 

def canonical_query_string(self, http_request):
        # POST requests pass parameters in through the
        # http_request.body field.
        if http_request.method == 'POST':
            return ""
        l = []
        for param in sorted(http_request.params):
            value = boto.utils.get_utf8_value(http_request.params[param])
            l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'),
                                urllib.parse.quote(value, safe='-_.~')))
        return '&'.join(l) 

def payload(self, http_request):
        body = http_request.body
        # If the body is a file like object, we can use
        # boto.utils.compute_hash, which will avoid reading
        # the entire body into memory.
        if hasattr(body, 'seek') and hasattr(body, 'read'):
            return boto.utils.compute_hash(body, hash_algorithm=sha256)[0]
        elif not isinstance(body, bytes):
            body = body.encode('utf-8')
        return sha256(body).hexdigest() 

def canonical_query_string(self, http_request):
        # Note that we just do not return an empty string for
        # POST request. Query strings in url are included in canonical
        # query string.
        l = []
        for param in sorted(http_request.params):
            value = boto.utils.get_utf8_value(http_request.params[param])
            l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'),
                                urllib.parse.quote(value, safe='-_.~')))
        return '&'.join(l) 

def _build_query_string(self, params):
        keys = list(params.keys())
        keys.sort(key=lambda x: x.lower())
        pairs = []
        for key in keys:
            val = boto.utils.get_utf8_value(params[key])
            pairs.append(key + '=' + self._escape_value(val.decode('utf-8')))
        return '&'.join(pairs) 

def _calc_signature(self, params, *args):
        boto.log.debug('using _calc_signature_0')
        hmac = self._get_hmac()
        s = params['Action'] + params['Timestamp']
        hmac.update(s.encode('utf-8'))
        keys = params.keys()
        keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
        pairs = []
        for key in keys:
            val = boto.utils.get_utf8_value(params[key])
            pairs.append(key + '=' + urllib.parse.quote(val))
        qs = '&'.join(pairs)
        return (qs, base64.b64encode(hmac.digest()))