Python jmespath.search() Examples

def get_data_member(parent, path):
    """
    Get a data member from a parent using a JMESPath search query,
    loading the parent if required. If the parent cannot be loaded
    and no data is present then an exception is raised.

    :type parent: ServiceResource
    :param parent: The resource instance to which contains data we
                   are interested in.
    :type path: string
    :param path: The JMESPath expression to query
    :raises ResourceLoadException: When no data is present and the
                                   resource cannot be loaded.
    :returns: The queried data or ``None``.
    """
    # Ensure the parent has its data loaded, if possible.
    if parent.meta.data is None:
        if hasattr(parent, 'load'):
            parent.load()
        else:
            raise ResourceLoadException(
                '{0} has no load method!'.format(parent.__class__.__name__))

    return jmespath.search(path, parent.meta.data) 

def get_body(self, field):
        """ extract body field with jmespath.
        """
        try:
            body = self.json
        except exceptions.JSONDecodeError:
            err_msg = u"Failed to extract body! => body.{}\n".format(field)
            err_msg += u"response body: {}\n".format(self.content)
            # logger.log_error(err_msg)
            raise exceptions.ExtractFailure(err_msg)

        if not field:
            # extract response body
            return body

        return jmespath.search(field, body) 

def build_iterator(self, **kwargs) -> List:
        results = []
        for feed_name, feed in self.feed_name_to_config.items():
            r = requests.get(
                url=feed.get('url', self.url),
                verify=self.verify,
                auth=self.auth,
                cert=self.cert,
                headers=self.headers,
                **kwargs
            )

            try:
                r.raise_for_status()
                data = r.json()
                result = jmespath.search(expression=feed.get('extractor'), data=data)
                results.append({feed_name: result})

            except ValueError as VE:
                raise ValueError(f'Could not parse returned data to Json. \n\nError massage: {VE}')

        return results 

def test_all_samples_any_affected_get_es_variants(self):
        search_model = VariantSearch.objects.create(search={
            'annotations': {'frameshift': ['frameshift_variant']}, 'inheritance': {'mode': 'any_affected'},
        })
        results_model = VariantSearchResults.objects.create(variant_search=search_model)
        results_model.families.set(Family.objects.filter(project__guid='R0001_1kg'))

        variants, total_results = get_es_variants(results_model, num_results=2)
        self.assertListEqual(variants, PARSED_ANY_AFFECTED_VARIANTS)
        self.assertEqual(total_results, 5)

        self.assertExecutedSearch(filters=[
            ANNOTATION_QUERY,
            {'bool': {
                'should': [
                    {'terms': {'samples_num_alt_1': ['HG00731', 'NA19675', 'NA20870']}},
                    {'terms': {'samples_num_alt_2': ['HG00731', 'NA19675', 'NA20870']}},
                    {'terms': {'samples': ['HG00731', 'NA19675', 'NA20870']}},
                ]
            }}
        ], sort=['xpos']) 

def get_data_member(parent, path):
    """
    Get a data member from a parent using a JMESPath search query,
    loading the parent if required. If the parent cannot be loaded
    and no data is present then an exception is raised.

    :type parent: ServiceResource
    :param parent: The resource instance to which contains data we
                   are interested in.
    :type path: string
    :param path: The JMESPath expression to query
    :raises ResourceLoadException: When no data is present and the
                                   resource cannot be loaded.
    :returns: The queried data or ``None``.
    """
    # Ensure the parent has its data loaded, if possible.
    if parent.meta.data is None:
        if hasattr(parent, 'load'):
            parent.load()
        else:
            raise ResourceLoadException(
                '{0} has no load method!'.format(parent.__class__.__name__))

    return jmespath.search(path, parent.meta.data) 

def test_multi_dataset_get_es_variants(self):
        search_model = VariantSearch.objects.create(search={'pathogenicity': {
            'clinvar': ['pathogenic'],
        }})
        results_model = VariantSearchResults.objects.create(variant_search=search_model)
        results_model.families.set(self.families)

        variants, _ = get_es_variants(results_model, num_results=5)
        self.assertListEqual(variants, [PARSED_SV_VARIANT] + PARSED_VARIANTS)
        path_filter = {'terms': {
            'clinvar_clinical_significance': [
                'Pathogenic', 'Pathogenic/Likely_pathogenic'
            ]
        }}
        self.assertExecutedSearches([
            dict(filters=[path_filter], start_index=0, size=5, sort=['xpos'], index=SV_INDEX_NAME),
            dict(filters=[path_filter, ALL_INHERITANCE_QUERY], start_index=0, size=5, sort=['xpos'], index=INDEX_NAME),
        ]) 

def get_data_member(parent, path):
    """
    Get a data member from a parent using a JMESPath search query,
    loading the parent if required. If the parent cannot be loaded
    and no data is present then an exception is raised.

    :type parent: ServiceResource
    :param parent: The resource instance to which contains data we
                   are interested in.
    :type path: string
    :param path: The JMESPath expression to query
    :raises ResourceLoadException: When no data is present and the
                                   resource cannot be loaded.
    :returns: The queried data or ``None``.
    """
    # Ensure the parent has its data loaded, if possible.
    if parent.meta.data is None:
        if hasattr(parent, 'load'):
            parent.load()
        else:
            raise ResourceLoadException(
                '{0} has no load method!'.format(parent.__class__.__name__))

    return jmespath.search(path, parent.meta.data) 

def process(self, resources):
        c = local_session(self.manager.session_factory).client('rds')
        for r in resources:
            param = {
                u['property']: u['value'] for u in self.data.get('update')
                if r.get(
                    u['property'],
                    jmespath.search(
                        self.conversion_map.get(u['property'], 'None'), r))
                    != u['value']}
            if not param:
                continue
            param['ApplyImmediately'] = self.data.get('immediate', False)
            param['DBInstanceIdentifier'] = r['DBInstanceIdentifier']
            try:
                c.modify_db_instance(**param)
            except c.exceptions.DBInstanceNotFoundFault:
                raise 

def handle_file_value(value, working_dir):
        components = value.split('|', 3)

        if len(components) == 3:
            url = components[2]
            return FileLoader.get_file(url, working_dir)
        elif len(components) == 4:
            url = components[2]
            pattern = components[3]
            file_content = FileLoader.get_yaml_or_json_file(url, working_dir)
            try:
                return jmespath.search(pattern, file_content)
            except JMESPathError as e:
                raise CfnSphereException(e)
        else:
            raise CfnSphereException("Invalid format for |File| macro, it must be |File|<path>[|<pattern>]") 

def _parse(self, data):
        """Parse a syslog string into a dictionary

        Matches syslog events with the following format:
            timestamp(Month DD HH:MM:SS) host application: message
        Example(s):
            Jan 10 19:35:33 vagrant-ubuntu-trusty-64 sudo: session opened for root
            Jan 10 19:35:13 vagrant-ubuntu-precise-32 ssh[13941]: login for mike

        Args:
            data (str): Data to be parsed

        Returns:
            list<tuple>: List of tuples with records and their parsing status
                Examples: [({'key': 'value'}, True)]
        """
        match = self._regex.search(data)
        if not match:
            return [(data, False)]

        return [(match.groupdict(), True)] 

def update_saved_search_handler(request, saved_search_guid):
    search = VariantSearch.objects.get(guid=saved_search_guid)
    if search.created_by != request.user:
        return create_json_response({}, status=403, reason='User does not have permission to edit this search')

    request_json = json.loads(request.body)
    name = request_json.pop('name', None)
    if not name:
        return create_json_response({}, status=400, reason='"Name" is required')

    search.name = name
    search.save()

    return create_json_response({
        'savedSearchesByGuid': {
            saved_search_guid: get_json_for_saved_search(search, request.user)
        }
    }) 

def process_cluster(self, client, cluster):
        try:
            config = dict(self.data.get('attributes'))
            modify = {}
            for k, v in config.items():
                if ((k in self.cluster_mapping and
                v != jmespath.search(self.cluster_mapping[k], cluster)) or
                v != cluster.get('PendingModifiedValues', {}).get(k, cluster.get(k))):
                    modify[k] = v
            if not modify:
                return

            modify['ClusterIdentifier'] = (cluster.get('PendingModifiedValues', {})
                                          .get('ClusterIdentifier')
                                          or cluster.get('ClusterIdentifier'))
            client.modify_cluster(**modify)
        except (client.exceptions.ClusterNotFoundFault):
            return
        except ClientError as e:
            self.log.warning(
                "Exception trying to modify cluster: %s error: %s",
                cluster['ClusterIdentifier'], e)
            raise 

def _json_path_records(self, payload):
        """Extract records from the original json payload using a provided JSON path

        Args:
            payload (dict): The parsed json data

        Returns:
            list: A list of JSON records extracted via JSON path
        """
        # Handle jsonpath extraction of records
        LOGGER.debug('Parsing records with JSONPath: %s', self._json_path)

        result = jmespath.search(self._json_path, payload)
        if not result:
            return []

        if not isinstance(result, list):
            result = [result]

        return result 

def process(self, resources):
        related_resources = []
        for rrid, r in zip(jmespath.search('[].[%s]' % self.data['key'], resources),
                           resources):
            related_resources.append((rrid[0], r))
        related_ids = {r[0] for r in related_resources}
        missing = False
        if None in related_ids:
            missing = True
            related_ids.discard(None)
        related_tag_map = self.get_resource_tag_map(self.data['resource'], related_ids)

        missing_related_tags = related_ids.difference(related_tag_map.keys())
        if not self.data.get('skip_missing', True) and (missing_related_tags or missing):
            raise PolicyExecutionError(
                "Unable to find all %d %s related resources tags %d missing" % (
                    len(related_ids), self.data['resource'],
                    len(missing_related_tags) + int(missing)))

        # rely on resource manager tag action implementation as it can differ between resources
        tag_action = self.manager.action_registry.get('tag')({}, self.manager)
        tag_action.id_key = tag_action.manager.get_model().id
        client = tag_action.get_client()

        stats = Counter()

        for related, r in related_resources:
            if (related is None or
                related in missing_related_tags or
                    not related_tag_map[related]):
                stats['missing'] += 1
            elif self.process_resource(
                    client, r, related_tag_map[related], self.data['tags'], tag_action):
                stats['tagged'] += 1
            else:
                stats['unchanged'] += 1

        self.log.info(
            'Tagged %d resources from related, missing-skipped %d unchanged %d',
            stats['tagged'], stats['missing'], stats['unchanged']) 

def process(self, resources):
        client = local_session(self.manager.session_factory).client('firehose')
        key = self.data.get('key_arn')
        for r in resources:
            if not r['DeliveryStreamStatus'] == 'ACTIVE':
                continue
            version = r['VersionId']
            name = r['DeliveryStreamName']
            d = r['Destinations'][0]
            destination_id = d['DestinationId']

            for dtype, dmetadata in self.DEST_MD.items():
                if dtype not in d:
                    continue
                dinfo = d[dtype]
                for k in dmetadata['clear']:
                    dinfo.pop(k, None)
                if dmetadata['encrypt_path']:
                    encrypt_info = jmespath.search(dmetadata['encrypt_path'], dinfo)
                else:
                    encrypt_info = dinfo
                encrypt_info.pop('NoEncryptionConfig', None)
                encrypt_info['KMSEncryptionConfig'] = {'AWSKMSKeyARN': key}

                for old_k, new_k in dmetadata['remap']:
                    if old_k in dinfo:
                        dinfo[new_k] = dinfo.pop(old_k)
                params = dict(DeliveryStreamName=name,
                              DestinationId=destination_id,
                              CurrentDeliveryStreamVersionId=version)
                params[dmetadata['update']] = dinfo
                client.update_destination(**params) 

def process(self, resources, event=None):
        instances = self.manager.get_resource_manager('ec2').resources()
        used = set(jmespath.search('[].KeyName', instances))
        if self.data.get('state', True):
            return [r for r in resources if r['KeyName'] not in used]
        else:
            return [r for r in resources if r['KeyName'] in used] 

def format_resource(self, r):
        envelope, payload = self.format_envelope(r)
        payload.update(self.filter_empty({
            'Name': r['name'],
            'EncryptionKey': r['encryptionKey'],
            'Environment': self.filter_empty({
                'Type': r['environment']['type'],
                'Certificate': r['environment'].get('certificate'),
                'RegistryCredential': self.filter_empty({
                    'Credential': jmespath.search(
                        'environment.registryCredential.credential', r),
                    'CredentialProvider': jmespath.search(
                        'environment.registryCredential.credentialProvider', r)
                }),
                'ImagePullCredentialsType': r['environment'].get(
                    'imagePullCredentialsType')
            }),
            'ServiceRole': r['serviceRole'],
            'VpcConfig': self.filter_empty({
                'VpcId': jmespath.search('vpcConfig.vpcId', r),
                'Subnets': jmespath.search('vpcConfig.subnets', r),
                'SecurityGroupIds': jmespath.search('vpcConfig.securityGroupIds', r)
            }),
            'Source': self.filter_empty({
                'Type': jmespath.search('source.type', r),
                'Location': jmespath.search('source.location', r),
                'GitCloneDepth': jmespath.search('source.gitCloneDepth', r)
            }),
        }))
        return envelope 

def get_ecs_cwe_sgs(self):
        sg_ids = set()
        expr = jmespath.compile(
            'EcsParameters.NetworkConfiguration.awsvpcConfiguration.SecurityGroups[]')
        for rule in self.manager.get_resource_manager(
                'event-rule-target').resources(augment=False):
            ids = expr.search(rule)
            if ids:
                sg_ids.update(ids)
        return sg_ids 

def process(self, resources, event=None):
        results = []
        accounts = self.get_accounts()
        owners = map(jmespath.compile, (
            'AccepterVpcInfo.OwnerId', 'RequesterVpcInfo.OwnerId'))

        for r in resources:
            for o_expr in owners:
                account_id = o_expr.search(r)
                if account_id and account_id not in accounts:
                    r.setdefault(
                        'c7n:CrossAccountViolations', []).append(account_id)
                    results.append(r)
        return results 

def save_query(self, parsed, **kwargs):
        """Saves the result of a JMESPath expression to a file.

        This method only saves the query data if the response code of
        the parsed result is < 300.
        """
        if is_parsed_result_successful(parsed):
            contents = jmespath.search(self.query, parsed)
            with open(self.value, 'w') as fp:
                # Don't write 'None' to a file -- write ''.
                if contents is None:
                    fp.write('')
                else:
                    fp.write(contents)
                os.chmod(self.value, self.perm) 

def get_service_name(self, name, search_filter):
        """Fetch account name as referenced by a particular service. """
        service_filter = "services[?name=='{}'].metadata.name".format(name)
        return one(jmespath.search(service_filter, self.get(search_filter))) 

def get_service(self, name, search_filter):
        """Fetch service metadata."""
        if self.version == 1:
            service_filter = "service.{name}".format(name=name)
            return jmespath.search(service_filter, self.get(search_filter))
        else:
            service_filter = "services[?name=='{}']".format(name)
            return one(jmespath.search(service_filter, self.get(search_filter))) 

def get_service_enabled(self, name, accounts_list=None, search_filter=None, region=None):
        """Get a list of accounts where a service has been enabled."""
        if not accounts_list:
            accounts = self.get_all(search_filter=search_filter)
        else:
            accounts = accounts_list

        if self.version == 1:
            accounts = accounts['accounts']

        enabled = []
        for account in accounts:
            if self.version == 1:
                account_filter = "accounts[?id=='{id}']".format(id=account['id'])
            else:
                account_filter = "[?id=='{id}']".format(id=account['id'])

            service = self.get_service(name, search_filter=account_filter)

            if self.version == 1:
                if service:
                    service = service['enabled']  # no region information available in v1
            else:
                if not region:
                    service_filter = "status[?enabled]"
                else:
                    service_filter = "status[?(region=='{region}' || region=='all') && enabled]".format(region=region)

                service = jmespath.search(service_filter, service)

            if service:
                enabled.append(account)

        return enabled 

def get_all(self, search_filter=None):
        """Fetch all data from backend."""
        items = self.backend.get_all()

        if not items:
            if self.version == 1:
                return {self.namespace: []}
            return []

        if search_filter:
            items = jmespath.search(search_filter, items)

        return items 

def remove_item(namespace, version, item, items):
    if version == 1:
        # NOTE only supports aws providers
        path = "{namespace}[?id!='{id}']".format(id=item['id'], namespace=namespace)
        return jmespath.search(path, items)
    else:
        return jmespath.search("[?id!='{id}']".format(id=item['id']), items) 

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('expression')
    parser.add_argument('-f', '--filename',
                        help=('The filename containing the input data.  '
                              'If a filename is not given then data is '
                              'read from stdin.'))
    parser.add_argument('--ast', action='store_true',
                        help=('Pretty print the AST, do not search the data.'))
    args = parser.parse_args()
    expression = args.expression
    if args.ast:
        # Only print the AST
        expression = jmespath.compile(args.expression)
        sys.stdout.write(pformat(expression.parsed))
        sys.stdout.write('\n')
        return 0
    if args.filename:
        with open(args.filename, 'r') as f:
            data = json.load(f)
    else:
        data = sys.stdin.read()
        data = json.loads(data)
    try:
        sys.stdout.write(json.dumps(
            jmespath.search(expression, data), indent=4))
        sys.stdout.write('\n')
    except exceptions.ArityError as e:
        sys.stderr.write("invalid-arity: %s\n" % e)
        return 1
    except exceptions.JMESPathTypeError as e:
        sys.stderr.write("invalid-type: %s\n" % e)
        return 1
    except exceptions.UnknownFunctionError as e:
        sys.stderr.write("unknown-function: %s\n" % e)
        return 1
    except exceptions.ParseError as e:
        sys.stderr.write("syntax-error: %s\n" % e)
        return 1 

def _parse(self, data):
        """Identify and extract nested payloads from parsed JSON records.

        Nested payloads can be detected with 'json_path' or 'json_regex_key' option.
        The value of these should be a JMESpath (http://jmespath.org/) compliant search
        expression that returns the desired nested records.

        If desired, fields present on the root of the record can be merged into child
        events using the 'envelope_keys' option.

        Args:
            data (dict): The raw json data loaded as a dictionary

        Returns:
            list<tuple>: List of tuples with records and their parsing status
                Examples: [({'key': 'value'}, True)]
        """
        # TODO (ryandeivert): migrate all of this to the base class and do away with JSONParser
        if not (self._json_path or self.json_regex_key):
            return [(data, True)]  # Nothing special to be done

        records = []
        if self._json_path:
            records = self._extract_via_json_path(data)
        elif self.json_regex_key:
            records = self._extract_via_json_regex_key(data)

        return records if records else [(data, False)] 

def _extract_via_json_regex_key(self, json_payload):
        """Extract records from the original json payload using the JSON configuration

        Args:
            json_payload (dict): The parsed json data

        Returns:
            list: A list of JSON records extracted via JSON path or regex
        """
        # Handle nested json object regex matching
        if not json_payload.get(self.json_regex_key):
            return False

        LOGGER.debug('Parsing records with JSON Regex Key')
        match = self._regex.search(str(json_payload[self.json_regex_key]))
        if not match:
            return False

        match_str = match.groups('json_blob')[0]
        try:
            record = json.loads(match_str)
            if not isinstance(record, dict):
                # purposely raising here to be caught below & handled
                raise TypeError('record data is not a dictionary')
        except (ValueError, TypeError) as err:
            LOGGER.debug('Matched regex string is invalid (%s): %s', str(err), match_str)
            return False

        return [(record, True)] 

def extract(self, extractors: Dict[Text, Text]) -> Dict[Text, Any]:
        if not extractors:
            return {}

        extract_mapping = {}
        for key, field in extractors.items():
            field_value = jmespath.search(field, self.resp_obj_meta)
            extract_mapping[key] = field_value

        logger.info(f"extract mapping: {extract_mapping}")
        return extract_mapping 

def _handleEvent(self, event):
          if 'CloudTrailEvent' not in event:
            logger.debug("No CloudTrailEvent key in event. Skipping")
            return None # ignore this situation

          ce_dict = json.loads(event['CloudTrailEvent'])

          if 'requestParameters' not in ce_dict:
            logger.debug("No requestParameters key in event['CloudTrailEvent']. Skipping")
            return None # ignore this situation

          rp_dict = ce_dict['requestParameters']

          import jmespath
          nodeType = jmespath.search('instanceType', rp_dict)
          numberOfNodes = jmespath.search('numberOfNodes', rp_dict)

          ts_obj = event['EventTime']
          # ts_obj = dt.datetime.utcfromtimestamp(ts_int)
          # ts_str = ts_obj.strftime('%Y-%m-%d %H:%M:%S')

          result = {
            'ServiceName': 'Redshift',
            'ResourceName': rp_dict['clusterIdentifier'],
            'EventTime': ts_obj, # ts_str,

            'EventName': self.eventName,
            'ResourceSize1': nodeType,
            'ResourceSize2': numberOfNodes,

          }

          return result