Python elasticsearch_dsl.Search() Examples

The following are 30 code examples of elasticsearch_dsl.Search(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module elasticsearch_dsl , or try the search function .
Example #1
Source File: facematch.py    From xi-iot with MIT License 6 votes vote down vote up
def run(self):
        s = Search(using=self.esclient, index=self.esIndex)
        count =0
        while True:
            try:
                #response = s.execute(True)
                response =[]
                for hit in s.scan():
                    response.append(hit)
                if count % 10 == 0:
                    count = 0
                    logging.info("Fetched registered faces from Elastic Search. Number of records found: %d",len(response))
                facematch.update_known_faces(response)
                count = count +1
            except Exception as e:
                logging.exception("Failed to get registered faces from Elastic Search.")
            # Sleep for 60 secs
            time.sleep(60) 
Example #2
Source File: esapi.py    From fooltrader with MIT License 6 votes vote down vote up
def es_get_accounts(main_chain='eos', user_id=None, start_vol=None, end_vol=None, from_idx=0, size=100,
                    order='totalEos', fields=None):
    index = '{}_account'.format(main_chain)

    if not fields:
        fields = ['id', 'timestamp', 'updateTimestamp', 'userId', 'totalEos', 'liquidEos', 'stackedEos',
                  'unstackingEos']

    if user_id:
        s = Search(using=es_client, index=index, doc_type='doc') \
            .filter('term', userId=user_id)
    elif start_vol and end_vol:
        range = {order: {'gte': start_vol, 'lt': end_vol}}
        s = Search(using=es_client, index=index, doc_type='doc') \
            .source(include=fields) \
            .filter('range', **range)
        s = s.sort({order: {"order": "desc"}})
    else:
        s = Search(using=es_client, index=index, doc_type='doc').source(include=fields)
        s = s.sort({order: {"order": "desc"}})

    resp = s[from_idx:from_idx + size].execute()

    return es_resp_to_payload(resp) 
Example #3
Source File: FeedElasticsearch.py    From content with MIT License 6 votes vote down vote up
def get_scan_generic_format(client, now, last_fetch_timestamp=None):
    """Gets a scan object in generic format"""
    # if method is simple date - convert the date string to datetime
    es = client.es
    time_field = client.time_field
    fetch_index = client.fetch_index
    if not fetch_index:
        fetch_index = '_all'
    if time_field:
        query = QueryString(query=time_field + ':*')
        range_field = {
            time_field: {'gt': last_fetch_timestamp, 'lte': now}} if last_fetch_timestamp else {
            time_field: {'lte': now}}
        search = Search(using=es, index=fetch_index).filter({'range': range_field}).query(query)
    else:
        search = Search(using=es, index=fetch_index).query(QueryString(query=client.query))
    return search 
Example #4
Source File: study_ceres_onion.py    From grimoirelab-elk with GNU General Public License v3.0 6 votes vote down vote up
def __list_uniques(self, date_range, field_name):
        """Retrieve a list of unique values in a given field within a date range.

        :param date_range:
        :param field_name:
        :return: list  of unique values.
        """
        # Get project list
        s = Search(using=self._es_conn, index=self._es_index)
        s = s.filter('range', **date_range)
        # from:to parameters (=> from: 0, size: 0)
        s = s[0:0]
        s.aggs.bucket('uniques', 'terms', field=field_name, size=1000)
        response = s.execute()
        uniques_list = []
        for item in response.aggregations.uniques.buckets:
            uniques_list.append(item.key)

        return uniques_list 
Example #5
Source File: esapi.py    From fooltrader with MIT License 6 votes vote down vote up
def es_get_statistic(security_item, the_date=None, start_date=None, end_date=None, level='day',
                     from_idx=0, size=500):
    security_item = to_security_item(security_item)

    index = get_es_statistic_index(security_type=security_item['type'], exchange=security_item['exchange'],
                                   level=level)
    # 单日的日k线直接按id获取
    if level == 'day' and the_date:
        doc_id = '{}_{}'.format(security_item['id'], to_time_str(the_date))
        return es_client.get_source(index=index, doc_type='doc', id=doc_id)
    elif start_date and end_date:
        s = Search(using=es_client, index=index, doc_type='doc') \
            .filter('term', code=security_item['code']) \
            .filter('range', timestamp={'gte': start_date, 'lte': end_date}) \
            .sort({"timestamp": {"order": "asc"}})

        resp = s[from_idx:from_idx + size].execute()

        return es_resp_to_payload(resp) 
Example #6
Source File: query.py    From udata with GNU Affero General Public License v3.0 6 votes vote down vote up
def search(self):
        """
        Construct the Search object.
        """
        s = Search(doc_type=self.doc_types, using=es.client,
                   index=es.index_name)
        # don't return any fields, just the metadata
        s = s.fields([])
        # Sort from parameters
        s = s.sort(*self.sorts)
        # Paginate from parameters
        s = s[self.page_start:self.page_end]
        # Same construction as parent class
        # Allows to give the same signature as simple search
        # ie. Response(data) instead of Response(search, data)
        return s.response_class(partial(SearchResult, self)) 
Example #7
Source File: FeedElasticsearch.py    From content with MIT License 6 votes vote down vote up
def get_scan_insight_format(client, now, last_fetch_timestamp=None, feed_type=None):
    """Gets a scan object in insight format"""
    time_field = client.time_field
    range_field = {
        time_field: {'gt': last_fetch_timestamp, 'lte': now}} if last_fetch_timestamp else {
        time_field: {'lte': now}}
    es = client.es
    query = QueryString(query=time_field + ":*")
    indices = client.fetch_index
    if feed_type == FEED_TYPE_CORTEX_MT:
        indices = '*-shared*'
        tenant_hash = demisto.getIndexHash()
        if tenant_hash:
            # all shared indexes minus this tenant shared
            indices += f',-*{tenant_hash}*-shared*'
    elif not indices:
        indices = '_all'
    search = Search(using=es, index=indices).filter({'range': range_field}).query(query)
    return search 
Example #8
Source File: models.py    From elasticsearch-django with MIT License 6 votes vote down vote up
def execute(
        cls,
        search: Search,
        search_terms: str = "",
        user: Optional[AbstractBaseUser] = None,
        reference: Optional[str] = "",
        save: bool = True,
    ) -> SearchQuery:
        """Create a new SearchQuery instance and execute a search against ES."""
        warnings.warn(
            "Deprecated - please use `execute_search` function instead.",
            DeprecationWarning,
        )
        return execute_search(
            search, search_terms=search_terms, user=user, reference=reference, save=save
        ) 
Example #9
Source File: models.py    From elasticsearch-django with MIT License 6 votes vote down vote up
def index_search_document(self, *, index: str) -> None:
        """
        Create or replace search document in named index.

        Checks the local cache to see if the document has changed,
        and if not aborts the update, else pushes to ES, and then
        resets the local cache. Cache timeout is set as "cache_expiry"
        in the settings, and defaults to 60s.

        """
        cache_key = self.search_document_cache_key
        new_doc = self.as_search_document(index=index)
        cached_doc = cache.get(cache_key)
        if new_doc == cached_doc:
            logger.debug("Search document for %r is unchanged, ignoring update.", self)
            return
        cache.set(cache_key, new_doc, timeout=get_setting("cache_expiry", 60))
        get_client().index(
            index=index,
            doc_type=self.search_doc_type,
            body=new_doc,
            id=self.pk,  # type: ignore
        ) 
Example #10
Source File: utils.py    From userline with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def get_statistics(index,pattern=None):
	conn = connections.get_connection()
	stats = {}
	fields = {
			'computer_name.keyword':'computers',
			'strings_parsed.source_user_name.keyword': 'srcuser',
			'strings_parsed.target_user_name.keyword': 'dstuser',
			'strings_parsed.target_machine_name.keyword': 'dstsrvname',
			'strings_parsed.target_machine_ip.keyword': 'dstsrvip',
		}
	scheme = {
			"size" : 0,
			"aggs" : {
			"count" : {
				"cardinality" : {
					"field" : None
					}
				}
			}
		}

	s = Search(using=conn,index=index)
	for f in fields.keys():
		s.aggs.bucket(fields[f],A('cardinality',field=f))
	resp = s.execute()
	res = resp.aggregations.to_dict()
	for agg in res.keys():
		stats[agg] = res[agg]['value']

	stats['total'] = resp['hits']['total']
	return stats 
Example #11
Source File: bollinger_band.py    From Mastering-Elasticsearch-7.0 with MIT License 6 votes vote down vote up
def bollinger_band(index='cf_etf_hist_price', start_date='2018-12-26', end_date='2019-03-25', symbol='rfem'):
    ESLowLevelClientByConnection.get_instance()
    search = Search(index=index, using='high_level_client')[0:0]
    search.query = Q(Bool(must=[Range(date={'gte': '2018-12-26', 'lte': '2019-03-25'}), Term(symbol='rfem')]))
    aggs = A(DateHistogram(field='date', interval='1d', format='yyyy-MM-dd', min_doc_count=1))
    aggs_tp = A(ScriptedMetric(init_script='state.totals=[]',
                map_script='state.totals.add((doc.high.value+doc.low.value+doc.close.value)/3)',
                combine_script='double total=0; for (t in state.totals) {total += t} return total',
                reduce_script='double total=0; for (t in states) {total += t} return total'))
    aggs_moving_avg = A(MovingAvg(model='simple', window=20, buckets_path='tp.value'))
    aggs_bbu = A(BucketScript(buckets_path={'SMA':'20_trading_days_moving_avg'}, script='params.SMA + 0.5'))
    aggs_bbl = A(BucketScript(buckets_path={'SMA': '20_trading_days_moving_avg'}, script='params.SMA - 0.5'))
    search.aggs.bucket('Bollinger_band', aggs).metric('tp', aggs_tp).pipeline(
        '20_trading_days_moving_avg', aggs_moving_avg).pipeline('BBU', aggs_bbu).pipeline('BBL', aggs_bbl)
    response = search.execute()
    print(response.to_dict()) 
Example #12
Source File: esnotifications.py    From stethoscope with Apache License 2.0 6 votes vote down vote up
def _get_notifications_by_email(self, email):
    search = elasticsearch_dsl.Search(using=self.client, index=self.config['ELASTICSEARCH_INDEX'],
      doc_type=self.config['ELASTICSEARCH_DOCTYPE'])

    query = self.create_query_for_email(search, email)

    # logger.debug("query:\n{!s}", pprint.pformat(query.to_dict()))

    try:
      response = query.execute()
    except elasticsearch.exceptions.ElasticsearchException:
      logger.exception("Exception caught in Elasticsearch query:\n  index: {!r}\n  doc_type: {!r}\n"
                       "  query: {!s}".format(self.config['ELASTICSEARCH_INDEX'],
                         self.config['ELASTICSEARCH_DOCTYPE'], pprint.pformat(query.to_dict())))

    # logger.debug("response:\n{!s}", pprint.pformat(response.to_dict()))

    return response.hits.hits 
Example #13
Source File: test_result.py    From elasticsearch-dsl-py with Apache License 2.0 6 votes vote down vote up
def test_interactive_helpers(dummy_response):
    res = response.Response(Search(), dummy_response)
    hits = res.hits
    h = hits[0]

    rhits = "[<Hit(test-index/elasticsearch): {}>, <Hit(test-index/42): {}...}}>, <Hit(test-index/47): {}...}}>, <Hit(test-index/53): {{}}>]".format(
        repr(dummy_response['hits']['hits'][0]['_source']),
        repr(dummy_response['hits']['hits'][1]['_source'])[:60],
        repr(dummy_response['hits']['hits'][2]['_source'])[:60],
    )

    assert res
    assert '<Response: %s>' % rhits == repr(res)
    assert rhits == repr(hits)
    assert {'meta', 'city', 'name'} == set(dir(h))
    assert "<Hit(test-index/elasticsearch): %r>" % dummy_response['hits']['hits'][0]['_source'] == repr(h) 
Example #14
Source File: log.py    From KubeOperator with Apache License 2.0 6 votes vote down vote up
def build_query(self, level, page, size, limit, keywords=None):
        s = Search(using=self.client, index=self.index_name)
        if level:
            ls = []
            for k in levels:
                if levels[k] <= levels[level]:
                    ls.append(k.lower())
            s = s.query("terms", levelname=ls)
        if page and size:
            s = s[(page - 1) * size:page * size]
        if keywords:
            s = s.query("match", message=keywords)
        if limit:
            now = datetime.now()
            start_time = now - timedelta(days=int(limit))
            s = s.query("range", timestamp={"gte": format_date(start_time), "lte": format_date(now)})
        s = s.sort({"timestamp": {"order": "desc", "unmapped_type": "date"}})
        return s 
Example #15
Source File: hook.py    From qb with MIT License 6 votes vote down vote up
def get_highlights(self, text):
        # query top 10 guesses
        s = Search(index='qb_ir_instance_of')[0:10].query('multi_match', query=text,
                fields=['wiki_content', 'qb_content', 'source_content'])
        s = s.highlight('qb_content').highlight('wiki_content')
        results = list(s.execute())
        guess = results[0] # take the best answer
        _highlights = guess.meta.highlight 
    
        try:
            wiki_content = list(_highlights.wiki_content)
        except AttributeError:
            wiki_content = None
    
        try:
            qb_content = list(_highlights.qb_content)
        except AttributeError:
            qb_content = None

        highlights = {'wiki': wiki_content,
                      'qb': qb_content,
                      'guess': guess.page}
        return highlights 
Example #16
Source File: bitshares_elasticsearch_client.py    From bitshares-explorer-api with MIT License 6 votes vote down vote up
def get_daily_volume(self, from_date, to_date):
        s = Search(using='operations', index="bitshares-*")
        s = s.extra(size=0)
        s = s.query('bool', filter = [
            Q('term', operation_type=4),
            Q('range', block_data__block_time={'gte': from_date, 'lte': to_date}),
            Q('term', operation_history__op_object__fill_price__quote__asset_id__keyword=config.CORE_ASSET_ID)
        ])

        a = A('date_histogram', field='block_data.block_time', interval='1d', format='yyyy-MM-dd') \
                .metric('volume', 'sum', field='operation_history.op_object.fill_price.quote.amount')
        s.aggs.bucket('volume_over_time', a)

        response = s.execute()

        daily_volumes = []
        for daily_volume in response.aggregations.volume_over_time.buckets:
            daily_volumes.append({ 'date': daily_volume.key_as_string, 'volume': daily_volume.volume.value })
        
        return daily_volumes 
Example #17
Source File: test_result.py    From elasticsearch-dsl-py with Apache License 2.0 6 votes vote down vote up
def test_iterating_over_response_gives_you_hits(dummy_response):
    res = response.Response(Search(), dummy_response)
    hits = list(h for h in res)

    assert res.success()
    assert 123 == res.took
    assert 4 == len(hits)
    assert all(isinstance(h, response.Hit) for h in hits)
    h = hits[0]

    assert 'test-index' == h.meta.index
    assert 'company' == h.meta.doc_type
    assert 'elasticsearch' == h.meta.id
    assert 12 == h.meta.score

    assert hits[1].meta.routing == 'elasticsearch' 
Example #18
Source File: es_wrapper.py    From bitshares-explorer-api with MIT License 6 votes vote down vote up
def get_trade_history(size=10, from_date='2015-10-10', to_date='now', sort_by='-operation_id_num',
                      search_after=None, base="1.3.0", quote="1.3.121"):

    s = Search(using=es, index="bitshares-*")

    s = s.extra(size=size)
    if search_after and search_after != '':
        s = s.extra(search_after=search_after.split(','))

    q = Q()
    q = q & Q("match", operation_type=4)
    q = q & Q("match", operation_history__op_object__is_maker=True)

    q = q & Q("match", operation_history__op_object__fill_price__base__asset_id=base)
    q = q & Q("match", operation_history__op_object__fill_price__quote__asset_id=quote)

    range_query = Q("range", block_data__block_time={'gte': from_date, 'lte': to_date})
    s.query = q & range_query

    s = s.sort(*sort_by.split(','))
    response = s.execute()
    verify_es_response(response)

    return [hit.to_dict() for hit in response] 
Example #19
Source File: test_result.py    From elasticsearch-dsl-py with Apache License 2.0 5 votes vote down vote up
def test_response_is_pickleable(dummy_response):
    res = response.Response(Search(), dummy_response)
    res.hits
    r = pickle.loads(pickle.dumps(res))

    assert r == res
    assert r._search == res._search
    assert r.hits == res.hits 
Example #20
Source File: test_result.py    From elasticsearch-dsl-py with Apache License 2.0 5 votes vote down vote up
def test_hit_is_pickleable(dummy_response):
    res = response.Response(Search(), dummy_response)
    hits = pickle.loads(pickle.dumps(res.hits))

    assert hits == res.hits
    assert hits[0].meta == res.hits[0].meta 
Example #21
Source File: test_result.py    From elasticsearch-dsl-py with Apache License 2.0 5 votes vote down vote up
def test_attribute_error_in_hits_is_not_hidden(dummy_response):
    def f(hit):
        raise AttributeError()

    s = Search().doc_type(employee=f)
    r = response.Response(s, dummy_response)
    with raises(TypeError):
        r.hits 
Example #22
Source File: test_result.py    From elasticsearch-dsl-py with Apache License 2.0 5 votes vote down vote up
def test_empty_response_is_false(dummy_response):
    dummy_response['hits']['hits'] = []
    res = response.Response(Search(), dummy_response)

    assert not res 
Example #23
Source File: study_ceres_onion.py    From grimoirelab-elk with GNU General Public License v3.0 5 votes vote down vote up
def __quarters(self, from_date=None):
        """Get a set of quarters with available items from a given index date.

        :param from_date:
        :return: list of `pandas.Period` corresponding to quarters
        """
        s = Search(using=self._es_conn, index=self._es_index)
        if from_date:
            # Work around to solve conversion problem of '__' to '.' in field name
            q = Q('range')
            q.__setattr__(self._sort_on_field, {'gte': from_date})
            s = s.filter(q)

        # from:to parameters (=> from: 0, size: 0)
        s = s[0:0]

        s.aggs.bucket(self.TIMEFRAME, 'date_histogram', field=self._timeframe_field,
                      interval='quarter', min_doc_count=1)
        response = s.execute()

        quarters = []
        for quarter in response.aggregations[self.TIMEFRAME].buckets:
            period = pandas.Period(quarter.key_as_string, 'Q')
            quarters.append(period)

        return quarters 
Example #24
Source File: utils.py    From userline with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def get_last_shutdown(index,maxtstamp,pattern):
	"""
	Look for the last shutdown event
	"""

	conn = connections.get_connection()

	q = [ \
		Q('match',data_type='windows:evtx:record') , \
		Q('match',event_identifier=config.EVENT_SHUTDOWN)
	]

	if pattern:
		q.append(Q('query_string',query=pattern,analyze_wildcard=True))

	s = Search(using=conn, index=index).query(Q('bool',must=q)).filter('range',datetime={'lte':maxtstamp}).sort('-datetime')[0:0]
	s.aggs.bucket('computer','terms',field='computer_name.keyword').bucket('shutdown','top_hits',size=1)

	res = s.execute()
	ret = {}
	for item in res.aggregations['computer']['buckets']:
		ret[item['key']] = item['shutdown']['hits']['hits'][0]

	if len(ret.keys()) == 0:
		ret = None

	return ret 
Example #25
Source File: utils.py    From userline with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def get_logout_event(index,logonid,timestamp,maxtstamp,screen):
	"""
	Look for the logoff event belonging to the given logon id or a shutdown event.
	"""
	conn = connections.get_connection()

	# workaround to fix time presition issues
	timestamp = timestamp - 999

	logoff = get_dsl_logoff_query(screen)
	q = [ \
		Q('match',data_type='windows:evtx:record') , \
		Q('match',xml_string=logonid) , \
		logoff \
	]

	s = Search(using=conn, index=index).query(Q('bool',must=q)).filter('range',datetime={'gte':timestamp,'lte':maxtstamp}).sort('-datetime')
	res = s.execute()
	try:
		evt = res[0]
	except:
		evt = None

	if evt is None:
		q = [ Q('match',event_identifier=config.EVENT_SHUTDOWN) ]
		s = Search(using=conn, index=index).query(Q('bool',must=q)).filter('range',datetime={'gte':timestamp,'lte':maxtstamp}).sort('-datetime')
		res = s.execute()
		try:
			evt = res[0]
		except:
			evt = None

	return evt 
Example #26
Source File: test_querysets.py    From django-zombodb with MIT License 5 votes vote down vote up
def test_dsl_search_cant_use_es_search(self):
        query = Search(index="my-index") \
            .filter("term", category="search") \
            .query("match", title="python")   \
            .exclude("match", description="beta")
        with self.assertRaises(InvalidElasticsearchQuery) as cm:
            Restaurant.objects.dsl_search(query, validate=True)
        self.assertEqual(
            str(cm.exception),
            "Do not use the `Search` class. "
            "`query` must be an instance of a class inheriting from `DslBase`.") 
Example #27
Source File: querysets.py    From django-zombodb with MIT License 5 votes vote down vote up
def dsl_search(
            self, query, validate=False, sort=False, score_attr='zombodb_score', limit=None):
        if isinstance(query, Search):
            raise InvalidElasticsearchQuery(
                "Do not use the `Search` class. "
                "`query` must be an instance of a class inheriting from `DslBase`.")

        query_dict = query.to_dict()

        return self.dict_search(
            query=query_dict,
            validate=validate,
            sort=sort,
            score_attr=score_attr,
            limit=limit) 
Example #28
Source File: bitshares_elasticsearch_client.py    From bitshares-explorer-api with MIT License 5 votes vote down vote up
def get_balances(self, account_id=None, asset_id=None):
        s = Search(using='objects', index="objects-balance")
        if account_id:
            s = s.filter('term', owner=account_id)
        if asset_id:
            s = s.filter('term', asset_type=asset_id)
        s = s.source([ 'owner_', 'balance', 'asset_type'])
        s = s.sort({ 'balance': { 'order': 'desc' } })
        s = s.params(clear_scroll=False) # Avoid calling DELETE on ReadOnly apis.

        balances = [hit.to_dict() for hit in s.scan()]
        for balance in balances:
            balance["owner"] = balance.pop("owner_")
        return balances 
Example #29
Source File: bitshares_elasticsearch_client.py    From bitshares-explorer-api with MIT License 5 votes vote down vote up
def get_accounts_with_referrer(self, account_id, size=20, from_=0):
        s = Search(using='objects', index="objects-account", extra={'size': size, 'from': from_})    \
                .filter('term', referrer__keyword=account_id)                                        \
                .source([
                    "id", "name", "referrer", 
                    "referrer_rewards_percentage", "lifetime_referrer", 
                    "lifetime_referrer_fee_percentage"])                            \
                .sort("name.keyword")

        response = s.execute()

        referrers = [hit.to_dict() for hit in response.hits]
        return (response.hits.total, referrers) 
Example #30
Source File: bitshares_elasticsearch_client.py    From bitshares-explorer-api with MIT License 5 votes vote down vote up
def get_asset_names(self, start):
        s = Search(using='objects', index="objects-asset") \
            .query('prefix', symbol__keyword=start)              \
            .source(['symbol'])
        s = s.params(clear_scroll=False) # Avoid calling DELETE on ReadOnly apis.

        asset_names = [ hit.symbol for hit in s.scan()]
        return asset_names