Python elasticsearch_dsl.Search() Examples
The following are 30
code examples of elasticsearch_dsl.Search().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
elasticsearch_dsl
, or try the search function
.
Example #1
Source File: facematch.py From xi-iot with MIT License | 6 votes |
def run(self): s = Search(using=self.esclient, index=self.esIndex) count =0 while True: try: #response = s.execute(True) response =[] for hit in s.scan(): response.append(hit) if count % 10 == 0: count = 0 logging.info("Fetched registered faces from Elastic Search. Number of records found: %d",len(response)) facematch.update_known_faces(response) count = count +1 except Exception as e: logging.exception("Failed to get registered faces from Elastic Search.") # Sleep for 60 secs time.sleep(60)
Example #2
Source File: esapi.py From fooltrader with MIT License | 6 votes |
def es_get_accounts(main_chain='eos', user_id=None, start_vol=None, end_vol=None, from_idx=0, size=100, order='totalEos', fields=None): index = '{}_account'.format(main_chain) if not fields: fields = ['id', 'timestamp', 'updateTimestamp', 'userId', 'totalEos', 'liquidEos', 'stackedEos', 'unstackingEos'] if user_id: s = Search(using=es_client, index=index, doc_type='doc') \ .filter('term', userId=user_id) elif start_vol and end_vol: range = {order: {'gte': start_vol, 'lt': end_vol}} s = Search(using=es_client, index=index, doc_type='doc') \ .source(include=fields) \ .filter('range', **range) s = s.sort({order: {"order": "desc"}}) else: s = Search(using=es_client, index=index, doc_type='doc').source(include=fields) s = s.sort({order: {"order": "desc"}}) resp = s[from_idx:from_idx + size].execute() return es_resp_to_payload(resp)
Example #3
Source File: FeedElasticsearch.py From content with MIT License | 6 votes |
def get_scan_generic_format(client, now, last_fetch_timestamp=None): """Gets a scan object in generic format""" # if method is simple date - convert the date string to datetime es = client.es time_field = client.time_field fetch_index = client.fetch_index if not fetch_index: fetch_index = '_all' if time_field: query = QueryString(query=time_field + ':*') range_field = { time_field: {'gt': last_fetch_timestamp, 'lte': now}} if last_fetch_timestamp else { time_field: {'lte': now}} search = Search(using=es, index=fetch_index).filter({'range': range_field}).query(query) else: search = Search(using=es, index=fetch_index).query(QueryString(query=client.query)) return search
Example #4
Source File: study_ceres_onion.py From grimoirelab-elk with GNU General Public License v3.0 | 6 votes |
def __list_uniques(self, date_range, field_name): """Retrieve a list of unique values in a given field within a date range. :param date_range: :param field_name: :return: list of unique values. """ # Get project list s = Search(using=self._es_conn, index=self._es_index) s = s.filter('range', **date_range) # from:to parameters (=> from: 0, size: 0) s = s[0:0] s.aggs.bucket('uniques', 'terms', field=field_name, size=1000) response = s.execute() uniques_list = [] for item in response.aggregations.uniques.buckets: uniques_list.append(item.key) return uniques_list
Example #5
Source File: esapi.py From fooltrader with MIT License | 6 votes |
def es_get_statistic(security_item, the_date=None, start_date=None, end_date=None, level='day', from_idx=0, size=500): security_item = to_security_item(security_item) index = get_es_statistic_index(security_type=security_item['type'], exchange=security_item['exchange'], level=level) # 单日的日k线直接按id获取 if level == 'day' and the_date: doc_id = '{}_{}'.format(security_item['id'], to_time_str(the_date)) return es_client.get_source(index=index, doc_type='doc', id=doc_id) elif start_date and end_date: s = Search(using=es_client, index=index, doc_type='doc') \ .filter('term', code=security_item['code']) \ .filter('range', timestamp={'gte': start_date, 'lte': end_date}) \ .sort({"timestamp": {"order": "asc"}}) resp = s[from_idx:from_idx + size].execute() return es_resp_to_payload(resp)
Example #6
Source File: query.py From udata with GNU Affero General Public License v3.0 | 6 votes |
def search(self): """ Construct the Search object. """ s = Search(doc_type=self.doc_types, using=es.client, index=es.index_name) # don't return any fields, just the metadata s = s.fields([]) # Sort from parameters s = s.sort(*self.sorts) # Paginate from parameters s = s[self.page_start:self.page_end] # Same construction as parent class # Allows to give the same signature as simple search # ie. Response(data) instead of Response(search, data) return s.response_class(partial(SearchResult, self))
Example #7
Source File: FeedElasticsearch.py From content with MIT License | 6 votes |
def get_scan_insight_format(client, now, last_fetch_timestamp=None, feed_type=None): """Gets a scan object in insight format""" time_field = client.time_field range_field = { time_field: {'gt': last_fetch_timestamp, 'lte': now}} if last_fetch_timestamp else { time_field: {'lte': now}} es = client.es query = QueryString(query=time_field + ":*") indices = client.fetch_index if feed_type == FEED_TYPE_CORTEX_MT: indices = '*-shared*' tenant_hash = demisto.getIndexHash() if tenant_hash: # all shared indexes minus this tenant shared indices += f',-*{tenant_hash}*-shared*' elif not indices: indices = '_all' search = Search(using=es, index=indices).filter({'range': range_field}).query(query) return search
Example #8
Source File: models.py From elasticsearch-django with MIT License | 6 votes |
def execute( cls, search: Search, search_terms: str = "", user: Optional[AbstractBaseUser] = None, reference: Optional[str] = "", save: bool = True, ) -> SearchQuery: """Create a new SearchQuery instance and execute a search against ES.""" warnings.warn( "Deprecated - please use `execute_search` function instead.", DeprecationWarning, ) return execute_search( search, search_terms=search_terms, user=user, reference=reference, save=save )
Example #9
Source File: models.py From elasticsearch-django with MIT License | 6 votes |
def index_search_document(self, *, index: str) -> None: """ Create or replace search document in named index. Checks the local cache to see if the document has changed, and if not aborts the update, else pushes to ES, and then resets the local cache. Cache timeout is set as "cache_expiry" in the settings, and defaults to 60s. """ cache_key = self.search_document_cache_key new_doc = self.as_search_document(index=index) cached_doc = cache.get(cache_key) if new_doc == cached_doc: logger.debug("Search document for %r is unchanged, ignoring update.", self) return cache.set(cache_key, new_doc, timeout=get_setting("cache_expiry", 60)) get_client().index( index=index, doc_type=self.search_doc_type, body=new_doc, id=self.pk, # type: ignore )
Example #10
Source File: utils.py From userline with BSD 3-Clause "New" or "Revised" License | 6 votes |
def get_statistics(index,pattern=None): conn = connections.get_connection() stats = {} fields = { 'computer_name.keyword':'computers', 'strings_parsed.source_user_name.keyword': 'srcuser', 'strings_parsed.target_user_name.keyword': 'dstuser', 'strings_parsed.target_machine_name.keyword': 'dstsrvname', 'strings_parsed.target_machine_ip.keyword': 'dstsrvip', } scheme = { "size" : 0, "aggs" : { "count" : { "cardinality" : { "field" : None } } } } s = Search(using=conn,index=index) for f in fields.keys(): s.aggs.bucket(fields[f],A('cardinality',field=f)) resp = s.execute() res = resp.aggregations.to_dict() for agg in res.keys(): stats[agg] = res[agg]['value'] stats['total'] = resp['hits']['total'] return stats
Example #11
Source File: bollinger_band.py From Mastering-Elasticsearch-7.0 with MIT License | 6 votes |
def bollinger_band(index='cf_etf_hist_price', start_date='2018-12-26', end_date='2019-03-25', symbol='rfem'): ESLowLevelClientByConnection.get_instance() search = Search(index=index, using='high_level_client')[0:0] search.query = Q(Bool(must=[Range(date={'gte': '2018-12-26', 'lte': '2019-03-25'}), Term(symbol='rfem')])) aggs = A(DateHistogram(field='date', interval='1d', format='yyyy-MM-dd', min_doc_count=1)) aggs_tp = A(ScriptedMetric(init_script='state.totals=[]', map_script='state.totals.add((doc.high.value+doc.low.value+doc.close.value)/3)', combine_script='double total=0; for (t in state.totals) {total += t} return total', reduce_script='double total=0; for (t in states) {total += t} return total')) aggs_moving_avg = A(MovingAvg(model='simple', window=20, buckets_path='tp.value')) aggs_bbu = A(BucketScript(buckets_path={'SMA':'20_trading_days_moving_avg'}, script='params.SMA + 0.5')) aggs_bbl = A(BucketScript(buckets_path={'SMA': '20_trading_days_moving_avg'}, script='params.SMA - 0.5')) search.aggs.bucket('Bollinger_band', aggs).metric('tp', aggs_tp).pipeline( '20_trading_days_moving_avg', aggs_moving_avg).pipeline('BBU', aggs_bbu).pipeline('BBL', aggs_bbl) response = search.execute() print(response.to_dict())
Example #12
Source File: esnotifications.py From stethoscope with Apache License 2.0 | 6 votes |
def _get_notifications_by_email(self, email): search = elasticsearch_dsl.Search(using=self.client, index=self.config['ELASTICSEARCH_INDEX'], doc_type=self.config['ELASTICSEARCH_DOCTYPE']) query = self.create_query_for_email(search, email) # logger.debug("query:\n{!s}", pprint.pformat(query.to_dict())) try: response = query.execute() except elasticsearch.exceptions.ElasticsearchException: logger.exception("Exception caught in Elasticsearch query:\n index: {!r}\n doc_type: {!r}\n" " query: {!s}".format(self.config['ELASTICSEARCH_INDEX'], self.config['ELASTICSEARCH_DOCTYPE'], pprint.pformat(query.to_dict()))) # logger.debug("response:\n{!s}", pprint.pformat(response.to_dict())) return response.hits.hits
Example #13
Source File: test_result.py From elasticsearch-dsl-py with Apache License 2.0 | 6 votes |
def test_interactive_helpers(dummy_response): res = response.Response(Search(), dummy_response) hits = res.hits h = hits[0] rhits = "[<Hit(test-index/elasticsearch): {}>, <Hit(test-index/42): {}...}}>, <Hit(test-index/47): {}...}}>, <Hit(test-index/53): {{}}>]".format( repr(dummy_response['hits']['hits'][0]['_source']), repr(dummy_response['hits']['hits'][1]['_source'])[:60], repr(dummy_response['hits']['hits'][2]['_source'])[:60], ) assert res assert '<Response: %s>' % rhits == repr(res) assert rhits == repr(hits) assert {'meta', 'city', 'name'} == set(dir(h)) assert "<Hit(test-index/elasticsearch): %r>" % dummy_response['hits']['hits'][0]['_source'] == repr(h)
Example #14
Source File: log.py From KubeOperator with Apache License 2.0 | 6 votes |
def build_query(self, level, page, size, limit, keywords=None): s = Search(using=self.client, index=self.index_name) if level: ls = [] for k in levels: if levels[k] <= levels[level]: ls.append(k.lower()) s = s.query("terms", levelname=ls) if page and size: s = s[(page - 1) * size:page * size] if keywords: s = s.query("match", message=keywords) if limit: now = datetime.now() start_time = now - timedelta(days=int(limit)) s = s.query("range", timestamp={"gte": format_date(start_time), "lte": format_date(now)}) s = s.sort({"timestamp": {"order": "desc", "unmapped_type": "date"}}) return s
Example #15
Source File: hook.py From qb with MIT License | 6 votes |
def get_highlights(self, text): # query top 10 guesses s = Search(index='qb_ir_instance_of')[0:10].query('multi_match', query=text, fields=['wiki_content', 'qb_content', 'source_content']) s = s.highlight('qb_content').highlight('wiki_content') results = list(s.execute()) guess = results[0] # take the best answer _highlights = guess.meta.highlight try: wiki_content = list(_highlights.wiki_content) except AttributeError: wiki_content = None try: qb_content = list(_highlights.qb_content) except AttributeError: qb_content = None highlights = {'wiki': wiki_content, 'qb': qb_content, 'guess': guess.page} return highlights
Example #16
Source File: bitshares_elasticsearch_client.py From bitshares-explorer-api with MIT License | 6 votes |
def get_daily_volume(self, from_date, to_date): s = Search(using='operations', index="bitshares-*") s = s.extra(size=0) s = s.query('bool', filter = [ Q('term', operation_type=4), Q('range', block_data__block_time={'gte': from_date, 'lte': to_date}), Q('term', operation_history__op_object__fill_price__quote__asset_id__keyword=config.CORE_ASSET_ID) ]) a = A('date_histogram', field='block_data.block_time', interval='1d', format='yyyy-MM-dd') \ .metric('volume', 'sum', field='operation_history.op_object.fill_price.quote.amount') s.aggs.bucket('volume_over_time', a) response = s.execute() daily_volumes = [] for daily_volume in response.aggregations.volume_over_time.buckets: daily_volumes.append({ 'date': daily_volume.key_as_string, 'volume': daily_volume.volume.value }) return daily_volumes
Example #17
Source File: test_result.py From elasticsearch-dsl-py with Apache License 2.0 | 6 votes |
def test_iterating_over_response_gives_you_hits(dummy_response): res = response.Response(Search(), dummy_response) hits = list(h for h in res) assert res.success() assert 123 == res.took assert 4 == len(hits) assert all(isinstance(h, response.Hit) for h in hits) h = hits[0] assert 'test-index' == h.meta.index assert 'company' == h.meta.doc_type assert 'elasticsearch' == h.meta.id assert 12 == h.meta.score assert hits[1].meta.routing == 'elasticsearch'
Example #18
Source File: es_wrapper.py From bitshares-explorer-api with MIT License | 6 votes |
def get_trade_history(size=10, from_date='2015-10-10', to_date='now', sort_by='-operation_id_num', search_after=None, base="1.3.0", quote="1.3.121"): s = Search(using=es, index="bitshares-*") s = s.extra(size=size) if search_after and search_after != '': s = s.extra(search_after=search_after.split(',')) q = Q() q = q & Q("match", operation_type=4) q = q & Q("match", operation_history__op_object__is_maker=True) q = q & Q("match", operation_history__op_object__fill_price__base__asset_id=base) q = q & Q("match", operation_history__op_object__fill_price__quote__asset_id=quote) range_query = Q("range", block_data__block_time={'gte': from_date, 'lte': to_date}) s.query = q & range_query s = s.sort(*sort_by.split(',')) response = s.execute() verify_es_response(response) return [hit.to_dict() for hit in response]
Example #19
Source File: test_result.py From elasticsearch-dsl-py with Apache License 2.0 | 5 votes |
def test_response_is_pickleable(dummy_response): res = response.Response(Search(), dummy_response) res.hits r = pickle.loads(pickle.dumps(res)) assert r == res assert r._search == res._search assert r.hits == res.hits
Example #20
Source File: test_result.py From elasticsearch-dsl-py with Apache License 2.0 | 5 votes |
def test_hit_is_pickleable(dummy_response): res = response.Response(Search(), dummy_response) hits = pickle.loads(pickle.dumps(res.hits)) assert hits == res.hits assert hits[0].meta == res.hits[0].meta
Example #21
Source File: test_result.py From elasticsearch-dsl-py with Apache License 2.0 | 5 votes |
def test_attribute_error_in_hits_is_not_hidden(dummy_response): def f(hit): raise AttributeError() s = Search().doc_type(employee=f) r = response.Response(s, dummy_response) with raises(TypeError): r.hits
Example #22
Source File: test_result.py From elasticsearch-dsl-py with Apache License 2.0 | 5 votes |
def test_empty_response_is_false(dummy_response): dummy_response['hits']['hits'] = [] res = response.Response(Search(), dummy_response) assert not res
Example #23
Source File: study_ceres_onion.py From grimoirelab-elk with GNU General Public License v3.0 | 5 votes |
def __quarters(self, from_date=None): """Get a set of quarters with available items from a given index date. :param from_date: :return: list of `pandas.Period` corresponding to quarters """ s = Search(using=self._es_conn, index=self._es_index) if from_date: # Work around to solve conversion problem of '__' to '.' in field name q = Q('range') q.__setattr__(self._sort_on_field, {'gte': from_date}) s = s.filter(q) # from:to parameters (=> from: 0, size: 0) s = s[0:0] s.aggs.bucket(self.TIMEFRAME, 'date_histogram', field=self._timeframe_field, interval='quarter', min_doc_count=1) response = s.execute() quarters = [] for quarter in response.aggregations[self.TIMEFRAME].buckets: period = pandas.Period(quarter.key_as_string, 'Q') quarters.append(period) return quarters
Example #24
Source File: utils.py From userline with BSD 3-Clause "New" or "Revised" License | 5 votes |
def get_last_shutdown(index,maxtstamp,pattern): """ Look for the last shutdown event """ conn = connections.get_connection() q = [ \ Q('match',data_type='windows:evtx:record') , \ Q('match',event_identifier=config.EVENT_SHUTDOWN) ] if pattern: q.append(Q('query_string',query=pattern,analyze_wildcard=True)) s = Search(using=conn, index=index).query(Q('bool',must=q)).filter('range',datetime={'lte':maxtstamp}).sort('-datetime')[0:0] s.aggs.bucket('computer','terms',field='computer_name.keyword').bucket('shutdown','top_hits',size=1) res = s.execute() ret = {} for item in res.aggregations['computer']['buckets']: ret[item['key']] = item['shutdown']['hits']['hits'][0] if len(ret.keys()) == 0: ret = None return ret
Example #25
Source File: utils.py From userline with BSD 3-Clause "New" or "Revised" License | 5 votes |
def get_logout_event(index,logonid,timestamp,maxtstamp,screen): """ Look for the logoff event belonging to the given logon id or a shutdown event. """ conn = connections.get_connection() # workaround to fix time presition issues timestamp = timestamp - 999 logoff = get_dsl_logoff_query(screen) q = [ \ Q('match',data_type='windows:evtx:record') , \ Q('match',xml_string=logonid) , \ logoff \ ] s = Search(using=conn, index=index).query(Q('bool',must=q)).filter('range',datetime={'gte':timestamp,'lte':maxtstamp}).sort('-datetime') res = s.execute() try: evt = res[0] except: evt = None if evt is None: q = [ Q('match',event_identifier=config.EVENT_SHUTDOWN) ] s = Search(using=conn, index=index).query(Q('bool',must=q)).filter('range',datetime={'gte':timestamp,'lte':maxtstamp}).sort('-datetime') res = s.execute() try: evt = res[0] except: evt = None return evt
Example #26
Source File: test_querysets.py From django-zombodb with MIT License | 5 votes |
def test_dsl_search_cant_use_es_search(self): query = Search(index="my-index") \ .filter("term", category="search") \ .query("match", title="python") \ .exclude("match", description="beta") with self.assertRaises(InvalidElasticsearchQuery) as cm: Restaurant.objects.dsl_search(query, validate=True) self.assertEqual( str(cm.exception), "Do not use the `Search` class. " "`query` must be an instance of a class inheriting from `DslBase`.")
Example #27
Source File: querysets.py From django-zombodb with MIT License | 5 votes |
def dsl_search( self, query, validate=False, sort=False, score_attr='zombodb_score', limit=None): if isinstance(query, Search): raise InvalidElasticsearchQuery( "Do not use the `Search` class. " "`query` must be an instance of a class inheriting from `DslBase`.") query_dict = query.to_dict() return self.dict_search( query=query_dict, validate=validate, sort=sort, score_attr=score_attr, limit=limit)
Example #28
Source File: bitshares_elasticsearch_client.py From bitshares-explorer-api with MIT License | 5 votes |
def get_balances(self, account_id=None, asset_id=None): s = Search(using='objects', index="objects-balance") if account_id: s = s.filter('term', owner=account_id) if asset_id: s = s.filter('term', asset_type=asset_id) s = s.source([ 'owner_', 'balance', 'asset_type']) s = s.sort({ 'balance': { 'order': 'desc' } }) s = s.params(clear_scroll=False) # Avoid calling DELETE on ReadOnly apis. balances = [hit.to_dict() for hit in s.scan()] for balance in balances: balance["owner"] = balance.pop("owner_") return balances
Example #29
Source File: bitshares_elasticsearch_client.py From bitshares-explorer-api with MIT License | 5 votes |
def get_accounts_with_referrer(self, account_id, size=20, from_=0): s = Search(using='objects', index="objects-account", extra={'size': size, 'from': from_}) \ .filter('term', referrer__keyword=account_id) \ .source([ "id", "name", "referrer", "referrer_rewards_percentage", "lifetime_referrer", "lifetime_referrer_fee_percentage"]) \ .sort("name.keyword") response = s.execute() referrers = [hit.to_dict() for hit in response.hits] return (response.hits.total, referrers)
Example #30
Source File: bitshares_elasticsearch_client.py From bitshares-explorer-api with MIT License | 5 votes |
def get_asset_names(self, start): s = Search(using='objects', index="objects-asset") \ .query('prefix', symbol__keyword=start) \ .source(['symbol']) s = s.params(clear_scroll=False) # Avoid calling DELETE on ReadOnly apis. asset_names = [ hit.symbol for hit in s.scan()] return asset_names