Python Examples of elasticsearch_dsl.connections.connections.create

Source File: __init__.py From resolwe with Apache License 2.0

6 votes

def prepare_connection():
    """Set dafault connection for ElasticSearch.

    .. warning::

        In case of using multiprocessing/multithreading, connection will
        be probably initialized in the main process/thread and the same
        connection (socket) will be used in all processes/threads. This
        will cause some unexpected timeouts of pushes to Elasticsearch.
        So make sure that this function is called again in each
        process/thread to make sure that unique connection will be used.
    """
    elasticsearch_host = getattr(settings, "ELASTICSEARCH_HOST", "localhost")
    elasticsearch_port = getattr(settings, "ELASTICSEARCH_PORT", 9200)
    connections.create_connection(
        hosts=["{}:{}".format(elasticsearch_host, elasticsearch_port)]
    )

Source File: ElasticBurp.py From WASE with GNU General Public License v3.0

6 votes

def applyConfig(self):
        try:
            print("Connecting to '%s', index '%s'" % (self.confESHost, self.confESIndex))
            self.es = connections.create_connection(hosts=[self.confESHost])
            self.idx = Index(self.confESIndex)
            self.idx.doc_type(DocHTTPRequestResponse)
            if self.idx.exists():
                self.idx.open()
            else:
                self.idx.create()
            self.callbacks.saveExtensionSetting("elasticburp.host", self.confESHost)
            self.callbacks.saveExtensionSetting("elasticburp.index", self.confESIndex)
            self.callbacks.saveExtensionSetting("elasticburp.tools", str(self.confBurpTools))
            self.callbacks.saveExtensionSetting("elasticburp.onlyresp", str(int(self.confBurpOnlyResp)))
        except Exception as e:
            JOptionPane.showMessageDialog(self.panel, "<html><p style='width: 300px'>Error while initializing ElasticSearch: %s</p></html>" % (str(e)), "Error", JOptionPane.ERROR_MESSAGE)

    ### ITab ###

Source File: __init__.py From darklight with Apache License 2.0

6 votes

def __enter__(self):
        connection = connections.create_connection(
            hosts=['{}:{}'.format(
                self.ini.read('ELASTICSEARCH', 'HOST'),
                self.ini.read('ELASTICSEARCH', 'PORT')
            )],
            http_auth="{}:{}".format(
                self.ini.read('ELASTICSEARCH', 'USERNAME'),
                self.ini.read('ELASTICSEARCH', 'PASSWORD')
            ),
        )

        # create when index not exist
        if not Webpage._index.exists():
            Webpage._index.create()

        if not Port._index.exists():
            Port._index.create()

        return connection

Source File: reindex_tokens.py From bearded-avenger with Mozilla Public License 2.0

6 votes

def restore_tokens():
    connections.create_connection(hosts=ES_NODES)
    Index(INDEX_NAME).delete()

    class Token(DocType):
        username = String()
        token = String()
        expires = Date()
        read = Boolean()
        write = Boolean()
        revoked = Boolean()
        acl = String()
        groups = String()
        admin = Boolean()
        last_activity_at = Date()

        class Meta:
            index = INDEX_NAME

    Token.init()
    reindex_results = connections.get_connection().reindex(body={"source": {"index": BACKUP_INDEX_NAME}, "dest": {"index": INDEX_NAME}}, request_timeout=3600)
    if reindex_results.get('created') + reindex_results.get('updated') == reindex_results.get('total'):
        return ('Tokens restored to previous schema successfully!')
    else:
        return ('Tokens did not restore from backup properly')

Source File: hook.py From qb with MIT License

5 votes

def __init__(self, game, call_every='step'):
        connections.create_connection(hosts=['localhost'])
        self.game = game
        self.call_every = call_every

Source File: WASEProxy.py From WASE with GNU General Public License v3.0

5 votes

def mitm_request(self, data):
	# Initialize ES connection and index
	res = connections.create_connection(hosts=[args.elasticsearch])
	idx = Index(args.index)
	idx.doc_type(DocHTTPRequestResponse)
	try:
	    DocHTTPRequestResponse.init()
	    idx.create()
	except:
	    pass

        r = HTTPRequest(data)

        # determine url
        if self.is_connect:
            scheme = "https"
        else:
            scheme = "http"
        url = scheme + "://" + self.hostname
        if scheme == "http" and int(self.port) != 80 or scheme == "https" and int(self.port) != 443:
            url += ":" + str(self.port)
        url += self.path

        if args.verbose:
            print(url)

        self.doc = DocHTTPRequestResponse(host=self.hostname, port=int(self.port), protocol=scheme)
        self.doc.meta.index = args.index
        self.doc.request.url = url
        self.doc.request.requestline = r.requestline
        self.doc.request.method = r.command
        self.doc.host = self.hostname
        self.doc.port = int(self.port)
        self.doc.protocol = scheme
            
        return data

Source File: elastic_logs.py From quay with Apache License 2.0

5 votes

def _initialize(self):
        """
        Initialize a connection to an ES cluster and creates an index template if it does not exist.
        """
        if not self._initialized:
            http_auth = None
            if self._access_key and self._secret_key and self._aws_region:
                http_auth = AWS4Auth(self._access_key, self._secret_key, self._aws_region, "es")
            elif self._access_key and self._secret_key:
                http_auth = (self._access_key, self._secret_key)
            else:
                logger.warn("Connecting to Elasticsearch without HTTP auth")

            self._client = connections.create_connection(
                hosts=[{"host": self._host, "port": self._port}],
                http_auth=http_auth,
                use_ssl=self._use_ssl,
                verify_certs=True,
                connection_class=RequestsHttpConnection,
                timeout=ELASTICSEARCH_DEFAULT_CONNECTION_TIMEOUT,
            )

            # Create a second connection with a timeout of 60s vs 10s.
            # For some reason the PUT template API can take anywhere between
            # 10s and 30s on the test cluster.
            # This only needs to be done once to initialize the index template
            connections.create_connection(
                alias=ELASTICSEARCH_TEMPLATE_CONNECTION_ALIAS,
                hosts=[{"host": self._host, "port": self._port}],
                http_auth=http_auth,
                use_ssl=self._use_ssl,
                verify_certs=True,
                connection_class=RequestsHttpConnection,
                timeout=ELASTICSEARCH_TEMPLATE_CONNECTION_TIMEOUT,
            )

            try:
                force_template_update = ELASTICSEARCH_FORCE_INDEX_TEMPLATE_UPDATE.lower() == "true"
                self._client.indices.get_template(self._index_prefix)
                LogEntry.init(
                    self._index_prefix,
                    self._index_settings,
                    skip_template_init=not force_template_update,
                )
            except NotFoundError:
                LogEntry.init(self._index_prefix, self._index_settings, skip_template_init=False)
            finally:
                try:
                    connections.remove_connection(ELASTICSEARCH_TEMPLATE_CONNECTION_ALIAS)
                except KeyError as ke:
                    logger.exception(
                        "Elasticsearch connection not found to remove %s: %s",
                        ELASTICSEARCH_TEMPLATE_CONNECTION_ALIAS,
                        ke,
                    )

            self._initialized = True

Source File: collector.py From politicos with GNU Affero General Public License v3.0

5 votes

def run(args):
    "Collect data"

    FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(format=FORMAT, level=args.log_level)
    # mute elasticsearch INFO logs
    log = logging.getLogger('elasticsearch')
    log.setLevel('ERROR')

    # Define a default ElasticSearch client
    es_hosts = [dict(host=args.es_host, port=args.es_port)]
    connections.create_connection(hosts=es_hosts, timeout=30)

    # Setup elastic search indices once before starting
    setup_index_template()

    if args.year:
        years = args.year.split(',')
    else:
        years = year_headers.keys()

    # Collect!
    for year in years:
        setup_index(year)
        tse = TSE(year, path=args.download_directory)
        tse.download_and_extract(remove_tmp_dir=False, remove_zip=False)
        tse.all_candidates()

Source File: __init__.py From bearded-avenger with Mozilla Public License 2.0

5 votes

def __init__(self, nodes=ES_NODES, **kwargs):

        if type(nodes) == str:
            nodes = nodes.split(',')

        if not nodes:
            nodes = ES_NODES

        self.indicators_prefix = kwargs.get('indicators_prefix', 'indicators')
        self.tokens_prefix = kwargs.get('tokens_prefix', 'tokens')

        logger.info('setting es nodes {}'.format(nodes))

        connections.create_connection(hosts=nodes)

        self._alive = False

        while not self._alive:
            if not self._health_check():
                logger.warn('ES cluster not accessible')
                logger.info('retrying connection in 30s')
                sleep(30)

            self._alive = True

        logger.info('ES connection successful')
        self.tokens = TokenManager()
        self.indicators = IndicatorManager()

Source File: reindex_tokens.py From bearded-avenger with Mozilla Public License 2.0

5 votes

def reindex_tokens():
    TokenBackup.init()
    connections.create_connection(hosts=ES_NODES)
    backup_results = connections.get_connection().reindex(body={"source": {"index": INDEX_NAME}, "dest": {"index": BACKUP_INDEX_NAME}}, request_timeout=3600)
    if backup_results.get('created') + backup_results.get('updated') == backup_results.get('total'):
        Index(INDEX_NAME).delete()
    else:
        return ('Tokens did not backup properly')
    time.sleep(1)
    Token.init()
    reindex_results = connections.get_connection().reindex(body={"source": {"index": BACKUP_INDEX_NAME}, "dest": {"index": INDEX_NAME}}, request_timeout=3600)
    if reindex_results.get('created') + reindex_results.get('updated') == reindex_results.get('total'):
        return ('Tokens reindexed successfully!')
    else:
        return ('Tokens did not reindex from backup properly')

Source File: zelasticsearch.py From csirtg-smrt-v1 with Mozilla Public License 2.0

5 votes

def __init__(self, remote='localhost:9200', index='indicators', **kwargs):
        super(_ElasticSearch, self).__init__(remote)

        self.index = index
        if isinstance(self.remote, str):
            self.remote = self.remote.split(',')

        connections.create_connection(hosts=self.remote)

Source File: connection.py From micromasters with BSD 3-Clause "New" or "Revised" License

4 votes

def get_conn(*, verify=True, verify_indices=None):
    """
    Lazily create the connection.

    Args:
        verify (bool): If true, check the presence of indices and mappings
        verify_indices (list of str): If set, check the presence of these indices. Else use the defaults.

    Returns:
        elasticsearch.client.Elasticsearch: An Elasticsearch client
    """
    # pylint: disable=global-statement
    global _CONN
    global _CONN_VERIFIED

    do_verify = False
    if _CONN is None:
        http_auth = settings.ELASTICSEARCH_HTTP_AUTH
        use_ssl = http_auth is not None
        _CONN = connections.create_connection(
            hosts=[settings.ELASTICSEARCH_URL],
            http_auth=http_auth,
            use_ssl=use_ssl,
            # make sure we verify SSL certificates (off by default)
            verify_certs=use_ssl
        )
        # Verify connection on first connect if verify=True.
        do_verify = verify

    if verify and not _CONN_VERIFIED:
        # If we have a connection but haven't verified before, do it now.
        do_verify = True

    if not do_verify:
        if not verify:
            # We only skip verification if we're reindexing or
            # deleting the index. Make sure we verify next time we connect.
            _CONN_VERIFIED = False
        return _CONN

    # Make sure everything exists.
    if verify_indices is None:
        verify_indices = set()
        for index_type in ALL_INDEX_TYPES:
            verify_indices = verify_indices.union(
                get_aliases(index_type)
            )
    for verify_index in verify_indices:
        if not _CONN.indices.exists(verify_index):
            raise ReindexException("Unable to find index {index_name}".format(
                index_name=verify_index
            ))

    _CONN_VERIFIED = True
    return _CONN

Python elasticsearch_dsl.connections.connections.create_connection() Examples