Python elasticsearch_dsl.connections.connections.get_connection() Examples

def reindex(doc_class, index, using, options):
    """
    Index all the things, using ElasticSearch's bulk API for speed.
    """
    def get_actions():
        for doc in doc_class.documents(cursor=options['cursor']):
            action = {
                '_index': index,
                '_type': doc_class._doc_type.name,
            }
            action.update(doc)
            yield action
    es = connections.get_connection(using)
    actions = get_actions() if options['quiet'] else progress(get_actions(), count=doc_class.count(), label=doc_class.__name__)
    bulk(es, actions)
    es.indices.refresh(index=index) 

def restore_tokens():
    connections.create_connection(hosts=ES_NODES)
    Index(INDEX_NAME).delete()

    class Token(DocType):
        username = String()
        token = String()
        expires = Date()
        read = Boolean()
        write = Boolean()
        revoked = Boolean()
        acl = String()
        groups = String()
        admin = Boolean()
        last_activity_at = Date()

        class Meta:
            index = INDEX_NAME

    Token.init()
    reindex_results = connections.get_connection().reindex(body={"source": {"index": BACKUP_INDEX_NAME}, "dest": {"index": INDEX_NAME}}, request_timeout=3600)
    if reindex_results.get('created') + reindex_results.get('updated') == reindex_results.get('total'):
        return ('Tokens restored to previous schema successfully!')
    else:
        return ('Tokens did not restore from backup properly') 

def store():
    try:
        connections.get_connection().indices.delete(index='indicators-*')
        connections.get_connection().indices.delete(index='tokens')
    except Exception as e:
        pass

    with Store(store_type='elasticsearch', nodes='127.0.0.1:9200') as s:
        s._load_plugin(nodes='127.0.0.1:9200')
        yield s

    try:
        assert connections.get_connection().indices.delete(index='indicators-*')
        assert connections.get_connection().indices.delete(index='tokens')
    except Exception:
        pass 

def store():
    try:
        connections.get_connection().indices.delete(index='indicators-*')
        connections.get_connection().indices.delete(index='tokens')
    except Exception as e:
        pass

    with Store(store_type='elasticsearch', nodes='127.0.0.1:9200') as s:
        s._load_plugin(nodes='127.0.0.1:9200')
        yield s

    try:
        assert connections.get_connection().indices.delete(index='indicators-*')
        assert connections.get_connection().indices.delete(index='tokens')
    except Exception:
        pass 

def store():
    try:
        connections.get_connection().indices.delete(index='indicators-*')
        connections.get_connection().indices.delete(index='tokens')
    except Exception as e:
        pass

    with Store(store_type='elasticsearch', nodes='127.0.0.1:9200') as s:
        s._load_plugin(nodes='127.0.0.1:9200')
        yield s

    try:
        assert connections.get_connection().indices.delete(index='indicators-*')
        assert connections.get_connection().indices.delete(index='tokens')
    except Exception:
        pass 

def push(self):
        """Push built documents to ElasticSearch."""
        self._refresh_connection()
        self.create_mapping()

        if not self.push_queue:
            logger.debug("No documents to push, skipping push.")
            return

        logger.debug(
            "Found %s documents to push to Elasticsearch.", len(self.push_queue)
        )

        bulk(
            connections.get_connection(),
            (doc.to_dict(True) for doc in self.push_queue),
            refresh=True,
        )
        self.push_queue = []

        logger.debug("Finished pushing builded documents to Elasticsearch server.") 

def flush(request, domain):
    # Should use the delete-by-query plugin
    # http://blog.appliedinformaticsinc.com/how-to-delete-elasticsearch-data-records-by-dsl-query/ # NOQA
    # Or the new API
    # https://www.elastic.co/guide/en/elasticsearch/reference/5.1/docs-delete-by-query.html  # NOQA
    # Perhaps we can use
    # connections.get_connection().delete_by_query ?!?!
    assert domain
    t0 = time.time()
    search = TitleDoc.search()
    search = search.filter('term', domain=domain.name)
    ids = set()
    for hit in search.scan():
        ids.add(hit._id)
    for _id in ids:
        TitleDoc.get(id=_id).delete()
    t1 = time.time()
    return http.JsonResponse({
        'messsage': 'OK',
        'took': t1 - t0,
    }) 

def handle(self, *args, **options):
        if not options['filename']:
            raise CommandError('Please specify a file (-f) to read data from')

        refresh_indices = set()

        def get_actions():
            for data in json.load(open(options['filename'], 'rb')):
                if options['index']:
                    data['_index'] = options['index']
                refresh_indices.add(data['_index'])
                yield data

        es = connections.get_connection()
        bulk(es, get_actions())

        for index in refresh_indices:
            es.indices.refresh(index=index) 

def delete(obj, index=None, using=None):
    """
    Shortcut to delete a Django object from the ES index based on it's model class.
    """
    from django.contrib.contenttypes.models import ContentType
    model_class = ContentType.objects.get_for_model(obj).model_class()
    for doc_class in model_documents.get(model_class, []):
        doc_using = using or doc_class._doc_type.using or 'default'
        doc_index = index or doc_class._doc_type.index or getattr(settings, 'SEEKER_INDEX', 'seeker')
        es = connections.get_connection(doc_using)
        try:
            es.delete(
                index=doc_index,
                doc_type=doc_class._doc_type.name,
                id=doc_class.get_id(obj),
                refresh=True
            )
        except NotFoundError:
            # If this object wasn't indexed for some reason (maybe not in the document's queryset), no big deal.
            pass 

def index(obj, index=None, using=None):
    """
    Shortcut to index a Django object based on it's model class.
    """
    from django.contrib.contenttypes.models import ContentType
    model_class = ContentType.objects.get_for_model(obj).model_class()
    for doc_class in model_documents.get(model_class, []):
        if not doc_class.queryset().filter(pk=obj.pk).exists():
            continue
        doc_using = using or doc_class._doc_type.using or 'default'
        doc_index = index or doc_class._doc_type.index or getattr(settings, 'SEEKER_INDEX', 'seeker')
        es = connections.get_connection(doc_using)
        body = doc_class.serialize(obj)
        doc_id = body.pop('_id', None)
        es.index(
            index=doc_index,
            doc_type=doc_class._doc_type.name,
            body=body,
            id=doc_id,
            refresh=True
        ) 

def __init__(self, node: Node, task: IndexMetadataTask, index: str):
        self.node = node
        self.task = task
        self.index_name = index
        self.elastic: Elasticsearch = connections.get_connection()

        if not self.elastic.indices.exists(self.index_name):
            init_index(self.index_name)

        self.fields_meta = {}
        self.init_fields_meta_cache()
        try:
            data_json = DataJson(node.catalog_url)
            themes = data_json.get('themeTaxonomy', [])
            self.themes = self.get_themes(themes)
        except Exception:
            raise ValueError("Error de lectura de los themes del catálogo") 

def setUpClass(cls):
        super(ViewTests, cls).setUpClass()
        es_client = connections.get_connection()
        if es_client.indices.exists(cls.index):
            es_client.indices.delete(cls.index)
        es_client.indices.create(cls.index, body=INDEX_CREATION_BODY)

        cls.catalog_id = 'csv_dump_test_catalog'
        path = os.path.join(samples_dir, 'distribution_daily_periodicity.json')
        index_catalog(cls.catalog_id, path, cls.index)
        cls.task = GenerateDumpTask()
        cls.task.save()
        gen = DumpGenerator(cls.task)
        gen.generate()

        DumpGenerator(cls.task, cls.catalog_id).generate() 

def get_statistics(index,pattern=None):
	conn = connections.get_connection()
	stats = {}
	fields = {
			'computer_name.keyword':'computers',
			'strings_parsed.source_user_name.keyword': 'srcuser',
			'strings_parsed.target_user_name.keyword': 'dstuser',
			'strings_parsed.target_machine_name.keyword': 'dstsrvname',
			'strings_parsed.target_machine_ip.keyword': 'dstsrvip',
		}
	scheme = {
			"size" : 0,
			"aggs" : {
			"count" : {
				"cardinality" : {
					"field" : None
					}
				}
			}
		}

	s = Search(using=conn,index=index)
	for f in fields.keys():
		s.aggs.bucket(fields[f],A('cardinality',field=f))
	resp = s.execute()
	res = resp.aggregations.to_dict()
	for agg in res.keys():
		stats[agg] = res[agg]['value']

	stats['total'] = resp['hits']['total']
	return stats 

def clear(cls, index=None, using=None):
        """
        Deletes the Elasticsearch mapping associated with this document type.
        """
        using = using or cls._doc_type.using or 'default'
        index = index or cls._doc_type.index or getattr(settings, 'SEEKER_INDEX', 'seeker')
        es = connections.get_connection(using)
        if es.indices.exists_type(index=index, doc_type=cls._doc_type.name):
            def get_actions():
                for hit in scan(es, index=index, doc_type=cls._doc_type.name, query={'query': {'match_all': {}}}):
                    yield {
                        '_op_type': 'delete',
                        '_index': index,
                        '_type': cls._doc_type.name,
                        '_id': hit['_id'],
                    }
            bulk(es, get_actions())
            es.indices.refresh(index=index) 

def __init__(self, index: str):
        self.elastic: Elasticsearch = connections.get_connection()
        self.index_name = index
        self.index = tseries_index(index) 

def __init__(self, task):
        self.elastic: Elasticsearch = connections.get_connection()
        self.task = task 

def tearDown(self) -> None:
        elastic = connections.get_connection()
        if elastic.indices.exists(self.index_name):
            elastic.indices.delete(self.index_name) 

def __init__(self, index: str = SERIES_QUERY_INDEX_NAME):
        self.es_index = Index(index)
        self.es_index.doc_type(SeriesQuery)
        self.es_connection = connections.get_connection() 

def tearDownClass(cls):
        super(ViewTests, cls).tearDownClass()
        connections.get_connection().indices.delete(cls.index)
        DumpFile.objects.all().delete()
        Node.objects.all().delete() 

def _create_index(self):
        dt = datetime.utcnow()
        dt = dt.strftime('%Y.%m')
        es = connections.get_connection()
        if not es.indices.exists('indicators-{}'.format(dt)):
            index = Index('indicators-{}'.format(dt))
            index.aliases(live={})
            index.doc_type(Indicator)
            index.create()

            m = Mapping('indicator')
            m.field('indicator_ipv4', 'ip')
            m.field('indicator_ipv4_mask', 'integer')
            m.save('indicators-{}'.format(dt))
        return 'indicators-{}'.format(dt) 

def test_smrt_elastcisearch():
    with Smrt(remote=REMOTE, client='elasticsearch') as s:
        assert type(s) is Smrt

        x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners')
        assert len(x) > 0

        x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners')
        assert len(x) > 0

        # cleanup
        es = connections.get_connection()
        cli = elasticsearch.client.IndicesClient(es)
        cli.delete(index='indicators-*') 

def index_catalog(catalog_id, catalog_path, index, node=None):
    """Indexa un catálogo. Útil para tests"""
    node = parse_catalog(catalog_id, catalog_path, node)

    index_task = mgmt.IndexDataTask.objects.create()
    for distribution in Distribution.objects.filter(dataset__catalog__identifier=catalog_id):
        index_distribution(distribution.identifier, node.id, index_task.id, index=index, force=True)

        for field in distribution.field_set.all():
            for key in meta_keys.HITS_KEYS:
                field.enhanced_meta.create(key=key, value=0)

    es_client = connections.get_connection()
    if es_client.indices.exists(index=index):
        es_client.indices.forcemerge(index=index) 

def __init__(self):
        self.prev_values = []
        self.elastic = connections.get_connection()
        self.bulk_items = [] 

def tearDownClass(cls):
        connections.get_connection().indices.delete(cls.index)
        super(DtaGeneratorTests, cls).tearDownClass() 

def __init__(self, *args, **kwargs):
        super(IndicatorManager, self).__init__(*args, **kwargs)

        self.indicators_prefix = kwargs.get('indicators_prefix', 'indicators')
        self.partition = PARTITION
        self.idx = self._current_index()
        self.last_index_check = datetime.now() - timedelta(minutes=5)
        self.last_index_value = None
        self.handle = connections.get_connection()
        self.lockm = LockManager(self.handle, logger)

        self._create_index() 

def tearDownClass(cls):
        super(SQLGeneratorTests, cls).tearDownClass()
        connections.get_connection().indices.delete(cls.index) 

def delete_if_exists(index):
    elastic = connections.get_connection()
    if elastic.indices.exists(index):
        elastic.indices.delete(index) 

def tearDown(self) -> None:
        elastic = connections.get_connection()
        if elastic.indices.exists(self.index):
            elastic.indices.delete(self.index) 

def handle(self, *args, **options):
        index = options['index'] or getattr(settings, 'SEEKER_INDEX', 'seeker')
        connection = options['using'] or 'default'
        es = connections.get_connection(connection)
        
        print 'Attempting to drop index "%s" using "%s" connection...' % (index, connection)
        if es.indices.exists(index=index):
            es.indices.delete(index=index)
            if es.indices.exists(index=index):
                print '...The index was NOT dropped.'
            else:
                print '...The index was dropped.'
        else:
            print '...The index could not be dropped because it does not exist.' 

def destroy(self):
        """Destroy an index."""
        self._refresh_connection()

        self.push_queue = []
        index_name = self.document_class()._get_index()
        connections.get_connection().indices.delete(index_name, ignore=404)

        self._mapping_created = False