Python elasticsearch.exceptions.ConnectionError() Examples

def get_test_client(nowait=False, **kwargs):
    # construct kwargs from the environment
    kw = {'timeout': 30}
    if 'TEST_ES_CONNECTION' in os.environ:
        from elasticsearch import connection
        kw['connection_class'] = getattr(connection, os.environ['TEST_ES_CONNECTION'])

    kw.update(kwargs)
    client = Elasticsearch([os.environ.get('TEST_ES_SERVER', {})], **kw)

    # wait for yellow status
    for _ in range(1 if nowait else 100):
        try:
            client.cluster.health(wait_for_status='yellow')
            return client
        except ConnectionError:
            time.sleep(.1)
    else:
        # timeout
        raise SkipTest("Elasticsearch failed to start.") 

def elastic_query(self, query: str, csv=False):
        """
        Request an http SQL query to elasticsearch
        """
        self.description = None
        # Sanitize query
        query = self.sanitize_query(query)
        payload = {"query": query}
        if csv:
            path = f"/{self.sql_path}/?format=csv"
        else:
            path = f"/{self.sql_path}/"
        try:
            resp = self.es.transport.perform_request("POST", path, body=payload)
        except es_exceptions.ConnectionError as e:
            raise exceptions.OperationalError(
                f"Error connecting to {self.url}: {e.info}"
            )
        except es_exceptions.RequestError as e:
            raise exceptions.ProgrammingError(
                f"Error ({e.error}): {e.info['error']['reason']}"
            )
        return resp 

def get_test_client(nowait=False, **kwargs):
    # construct kwargs from the environment
    kw = {"timeout": 30}
    if "TEST_ES_CONNECTION" in os.environ:
        from elasticsearch import connection

        kw["connection_class"] = getattr(connection, os.environ["TEST_ES_CONNECTION"])

    kw.update(kwargs)
    client = Elasticsearch([os.environ.get("TEST_ES_SERVER", {})], **kw)

    # wait for yellow status
    for _ in range(1 if nowait else 100):
        try:
            client.cluster.health(wait_for_status="yellow")
            return client
        except ConnectionError:
            time.sleep(0.1)
    else:
        # timeout
        raise SkipTest("Elasticsearch failed to start.") 

def connect(self):
        counter = 0
        self.success = False

        if self.username and self.password:
            url = "%s://%s:%s@%s:%d" % (self.protocol, self.username, self.password, self.host, self.port)
        else:
            url = "%s://%s:%d" % (self.protocol, self.host, self.port)

        while True:
            try:
                self.es = Elasticsearch([url])
                self.es.cluster.health(wait_for_status='green', request_timeout=20)
                self.success = True
                break
            except exceptions.ConnectionError as e:
                logger.warning('Still trying to connect to Elasticsearch...')
                counter += 1

                if counter == MAX_ATTEMPTS:
                    break

            logger.info('Sleeping 10 seconds...')
            time.sleep(10) 

def fetch_public_sources():
    proxy_params = get_system_settings().get_proxy_params()
    try:
        hdrs = { 'User-Agent': 'scirius' }
        if proxy_params:
            resp = requests.get(settings.DEFAULT_SOURCE_INDEX_URL, proxies = proxy_params, headers = hdrs)
        else:
            resp = requests.get(settings.DEFAULT_SOURCE_INDEX_URL, headers = hdrs)
        resp.raise_for_status()
    except requests.exceptions.ConnectionError, e:
        if "Name or service not known" in unicode(e):
            raise IOError("Connection error 'Name or service not known'")
        elif "Connection timed out" in unicode(e):
            raise IOError("Connection error 'Connection timed out'")
        else:
            raise IOError("Connection error '%s'" % (e)) 

def get_test_client(nowait=False):
    # construct kwargs from the environment
    kw = {}
    if 'TEST_ES_CONNECTION' in os.environ:
        from elasticsearch import connection
        kw['connection_class'] = getattr(connection, os.environ['TEST_ES_CONNECTION'])

    client = Elasticsearch([os.environ.get('TEST_ES_SERVER', {})], **kw)

    # wait for yellow status
    for _ in range(1 if nowait else 100):
        try:
            client.cluster.health(wait_for_status='yellow')
            return client
        except ConnectionError:
            time.sleep(.1)
    else:
        # timeout
        raise SkipTest("Elasticsearch failed to start.") 

def get_test_client(nowait=False):
    # construct kwargs from the environment
    kw = {}
    if 'TEST_ES_CONNECTION' in os.environ:
        from elasticsearch import connection
        kw['connection_class'] = getattr(connection, os.environ['TEST_ES_CONNECTION'])

    client = Elasticsearch([os.environ.get('TEST_ES_SERVER', {})], **kw)

    # wait for yellow status
    for _ in range(1 if nowait else 100):
        try:
            client.cluster.health(wait_for_status='yellow')
            return client
        except ConnectionError:
            time.sleep(.1)
    else:
        # timeout
        raise SkipTest("Elasticsearch failed to start.") 

def get(self, request, format=None):
        # TODO: catch all exception. At the very least, deal with 404 not found and
        # connection refused exceptions.
        # Temporarily remove exceptions for debugging.
        try:
            trail_ids = [x["key"] for x in self.es.search(index=self.index, body={
                "aggs" : {
                    "trail_id" : {
                        "terms" : { "field" : "trail_id" }
                    }
                }
            })["aggregations"]["trail_id"]["buckets"]]
            response = self.create_trails(trail_ids)
        except ConnectionError as e:
            raise OSError("Failed to connect to local elasticsearch instance.")
        except NotFoundError:
            raise DataWakeIndexUnavailable
        return Response(response) 

def get_array_type_columns(self, table_name: str) -> "Cursor":
        """
            Queries the index (table) for just one record
            and return a list of array type columns.
            This is useful since arrays are not supported by ES SQL
        """
        array_columns = []
        try:
            resp = self.es.search(index=table_name, size=1)
        except es_exceptions.ConnectionError as e:
            raise exceptions.OperationalError(
                f"Error connecting to {self.url}: {e.info}"
            )
        except es_exceptions.NotFoundError as e:
            raise exceptions.ProgrammingError(
                f"Error ({e.error}): {e.info['error']['reason']}"
            )
        try:
            _source = resp["hits"]["hits"][0]["_source"]
        except KeyError as e:
            raise exceptions.DataError(
                f"Error inferring array type columns {self.url}: {e}"
            )
        for col_name, value in _source.items():
            # If it's a list (ES Array add to cursor)
            if isinstance(value, list):
                if len(value) > 0:
                    # If it's an array of objects add all keys
                    if isinstance(value[0], dict):
                        for in_col_name in value[0]:
                            array_columns.append([f"{col_name}.{in_col_name}"])
                            array_columns.append([f"{col_name}.{in_col_name}.keyword"])
                        continue
                array_columns.append([col_name])
                array_columns.append([f"{col_name}.keyword"])
        # Not array column found
        if not array_columns:
            array_columns = [[]]
        self.description = [("name", Type.STRING, None, None, None, None, None)]
        self._results = array_columns
        return self 

def _health_check(self):
        try:
            x = connections.get_connection().cluster.health()
        except ConnectionError as e:
            logger.warn('elasticsearch connection error')
            logger.error(e)
            return

        except Exception as e:
            logger.error(traceback.print_exc())
            return

        logger.info('ES cluster is: %s' % x['status'])
        return x 

def load(self):
        """Loads the trained entity resolution model from disk."""
        try:
            if self._use_text_rel:
                scoped_index_name = get_scoped_index_name(
                    self._app_namespace, self._es_index_name
                )
                if not self._es_client.indices.exists(index=scoped_index_name):
                    self.fit()
            else:
                self.fit()

        except EsConnectionError as e:
            logger.error(
                "Unable to connect to Elasticsearch: %s details: %s", e.error, e.info
            )
            raise EntityResolverConnectionError(es_host=self._es_client.transport.hosts)
        except TransportError as e:
            logger.error(
                "Unexpected error occurred when sending requests to Elasticsearch: %s "
                "Status code: %s details: %s",
                e.error,
                e.status_code,
                e.info,
            )
            raise EntityResolverError
        except ElasticsearchException:
            raise EntityResolverError 

def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None):
        url_path = url
        if params:
            url_path = '%s?%s' % (url, urlencode(params or {}))
        url = self.base_url + url_path

        start = self.loop.time()
        response = None
        try:
            with async_timeout.timeout(timeout or self.timeout, loop=self.loop):
                response = yield from self.session.request(method, url, data=body, headers=headers)
                raw_data = yield from response.text()
            duration = self.loop.time() - start

        except asyncio.CancelledError:
            raise

        except Exception as e:
            self.log_request_fail(method, url, url_path, body, self.loop.time() - start, exception=e)
            if isinstance(e, ServerFingerprintMismatch):
                raise SSLError('N/A', str(e), e)
            if isinstance(e, asyncio.TimeoutError):
                raise ConnectionTimeout('TIMEOUT', str(e), e)
            raise ConnectionError('N/A', str(e), e)

        finally:
            if response is not None:
                yield from response.release()

        # raise errors based on http status codes, let the client handle those if needed
        if not (200 <= response.status < 300) and response.status not in ignore:
            self.log_request_fail(method, url, url_path, body, duration, status_code=response.status, response=raw_data)
            self._raise_error(response.status, raw_data)

        self.log_request_success(method, url, url_path, body, response.status, raw_data, duration)

        return response.status, response.headers, raw_data 

def handle(self, *args, **options):
        try:
            count = self.es_clear()
        except ConnectionError as e:
            self.stderr.write('Could not connect to Elasticsearch, please retry later.')
            raise CommandError(repr(e))
        if count:
            self.stdout.write('Data erased successfully (%i index%s deleted)' % (count, 'es' if count > 1 else ''))
        else:
            self.stdout.write('There is no index to erase') 

def post(self, request, format=None):
        es_data = ESData()
        msg = None

        try:
            es_data.es_clear()
        except ConnectionError as e:
            msg = 'Could not connect to Elasticsearch'
        except Exception as e:
            msg = 'Clearing failed: %s' % e

        if msg is not None:
            raise serializers.ValidationError({'delete_es_logs': [msg]})

        return Response({'delete_es_logs': 'ok'}) 

def elasticsearch():
    try:
        yield Elasticsearch(settings.HOOVER_ELASTICSEARCH_URL)
    except ConnectionError:
        raise SearchError('Could not connect to Elasticsearch.')
    except RequestError as e:
        reason = 'reason unknown'
        try:
            if e.info:
                reason = e.info['error']['root_cause'][0]['reason']
        except LookupError:
            pass
        raise SearchError('Elasticsearch failed: ' + reason) 

def delete_index(self, index_name):
        """Delete Elasticsearch index.

        Args:
            index_name: Name of the index to delete.
        """
        if self.client.indices.exists(index_name):
            try:
                self.client.indices.delete(index=index_name)
            except ConnectionError as e:
                raise RuntimeError(
                    'Unable to connect to Timesketch backend: {}'.format(e)
                ) 

def wait_until_responsive(self, timeout, clock=timeit.default_timer):
        """Wait until ElasticSearch becomes responsive (or too much time passes)."""

        # Elapsed time is a floating point number of seconds.
        timeout = timeout.total_seconds()

        # Don't poll unless we're asked to.
        if timeout <= 0.0:
            return

        # Periodically poll ElasticSearch.  Keep going until ElasticSearch is
        # responsive *and* the writeback index exists.
        ref = clock()
        while (clock() - ref) < timeout:
            try:
                if self.writeback_es.indices.exists(self.writeback_alias):
                    return
            except ConnectionError:
                pass
            time.sleep(1.0)

        if self.writeback_es.ping():
            logging.error(
                'Writeback alias "%s" does not exist, did you run `elastalert-create-index`?',
                self.writeback_alias,
            )
        else:
            logging.error(
                'Could not reach ElasticSearch at "%s:%d".',
                self.conf['es_host'],
                self.conf['es_port'],
            )
        exit(1) 

def init_connection(self):
        """
        Initialize the connection with Elasticsearch

        :return: Connection object if connection with Elasticsearch succeeded, False otherwise
        """
        try:
            http_auth = (self.settings.config.get("general", "es_username"),
                         self.settings.config.get("general", "es_password"))
        except NoOptionError:
            http_auth = ("", "")

        self.conn = Elasticsearch([self.settings.config.get("general", "es_url")],
                                  http_auth=http_auth,
                                  use_ssl=False,
                                  timeout=self.settings.config.getint("general", "es_timeout"),
                                  verify_certs=False,
                                  retry_on_timeout=True)
        try:
            self.conn.info()
            self.logging.logger.info("connected to Elasticsearch on host %s" %
                                     (self.settings.config.get("general", "es_url")))
            result = self.conn
        except AuthenticationException:
            self.logging.logger.error("could not connect to Elasticsearch on host %s due to authentication error." %
                                      (self.settings.config.get("general", "es_url")))
            result = False
        except ConnectionError:
            self.logging.logger.error("could not connect to Elasticsearch on host %s" %
                                      (self.settings.config.get("general", "es_url")))
            result = False

        return result 

def set_up_index(self):
        try:
            try:
                try:
                    index_exists = self.__es_conn__.indices.exists(index=__index_name__)
                    if not index_exists:
                        self.create_index()
                    else:
                        res = self.__es_conn__.indices.get_mapping(index=__index_name__)
                        try:
                            current_version = res[__index_name__]['mappings']['_meta']['version']
                            if current_version < __index_version__:
                                self.update_index(current_version)
                            elif current_version is None:
                                logger.error("Old Index Mapping. Manually reindex the index to persist your data.")
                                print("\n -- Old Index Mapping. Manually reindex the index to persist your data.--\n")
                                sys.exit(1)
                        except KeyError:
                            logger.error("Old Index Mapping. Manually reindex the index to persist your data.")
                            print("\n -- Old Index Mapping. Manually reindex the index to persist your data.--\n")
                            sys.exit(1)

                except ESConnectionError as e:
                    logger.error("Elasticsearch is not installed or its service is not running. {0}".format(e))
                    print("\n -- Elasticsearch is not installed or its service is not running.--\n", e)
                    sys.exit(1)
            except NewConnectionError:
                pass
        except ConnectionRefusedError:
            pass 

def create_index(self, index_name=uuid4().hex, doc_type='generic_event'):
        """Create index with Timesketch settings.

        Args:
            index_name: Name of the index. Default is a generated UUID.
            doc_type: Name of the document type. Default id generic_event.

        Returns:
            Index name in string format.
            Document type in string format.
        """
        _document_mapping = {
            'properties': {
                'timesketch_label': {
                    'type': 'nested'
                },
                'datetime': {
                    'type': 'date'
                }
            }
        }

        # TODO: Remove when we deprecate Elasticsearch version 6.x
        if self.version.startswith('6'):
            _document_mapping = {doc_type: _document_mapping}

        if not self.client.indices.exists(index_name):
            try:
                self.client.indices.create(
                    index=index_name, body={'mappings': _document_mapping})
            except ConnectionError:
                raise RuntimeError('Unable to connect to Timesketch backend.')
            except RequestError:
                index_exists = self.client.indices.exists(index_name)
                es_logger.warning(
                    'Attempting to create an index that already exists '
                    '({0:s} - {1:s})'.format(index_name, str(index_exists)))

        # We want to return unicode here to keep SQLalchemy happy.
        if six.PY2:
            if not isinstance(index_name, six.text_type):
                index_name = codecs.decode(index_name, 'utf-8')

            if not isinstance(doc_type, six.text_type):
                doc_type = codecs.decode(doc_type, 'utf-8')

        return index_name, doc_type 

def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None):
        url_path = url
        if params:
            query_string = urlencode(params)
        else:
            query_string = ""
        # Provide correct URL object to avoid string parsing in low-level code
        url = yarl.URL.build(scheme=self.scheme,
                             host=self.hostname,
                             port=self.port,
                             path=url,
                             query_string=query_string,
                             encoded=True)

        start = self.loop.time()
        response = None
        try:
            request_timeout = timeout or self.timeout.total
            with async_timeout.timeout(request_timeout, loop=self.loop):
                # override the default session timeout explicitly
                response = yield from self.session.request(method, url, data=body, headers=headers, timeout=request_timeout)
                raw_data = yield from response.text()
            duration = self.loop.time() - start

        except asyncio.CancelledError:
            raise

        except Exception as e:
            self.log_request_fail(method, url, url_path, body, self.loop.time() - start, exception=e)
            if isinstance(e, ServerFingerprintMismatch):
                raise SSLError('N/A', str(e), e)
            if isinstance(e, asyncio.TimeoutError):
                raise ConnectionTimeout('TIMEOUT', str(e), e)
            raise ConnectionError('N/A', str(e), e)

        finally:
            if response is not None:
                yield from response.release()

        # raise errors based on http status codes, let the client handle those if needed
        if not (200 <= response.status < 300) and response.status not in ignore:
            self.log_request_fail(method, url, url_path, body, duration, status_code=response.status, response=raw_data)
            self._raise_error(response.status, raw_data)

        self.log_request_success(method, url, url_path, body, response.status, raw_data, duration)

        return response.status, response.headers, raw_data 

def wait_and_configure(self):
        for i in range(self.MAX_CONNECTION_ATTEMPTS):
            # get or create index
            try:
                info = self._es.info()
                self.version = [int(i) for i in info["version"]["number"].split(".")]
                if not self._es.indices.exists(self.index):
                    self._es.indices.create(self.index, body=self.get_index_conf())
                    self.use_mapping_types = False
                    logger.info("Index %s created", self.index)
            except ConnectionError:
                s = (i + 1) * random.uniform(0.9, 1.1)
                logger.warning('Could not connect to server %d/%d. Sleep %ss',
                               i + 1, self.MAX_CONNECTION_ATTEMPTS, s)
                time.sleep(s)
                continue
            except RequestError as exception:
                error = exception.error.lower()
                if "already" in error and "exist" in error:
                    # race
                    logger.info('Index %s exists', self.index)
                else:
                    raise
            # wait for index recovery
            waiting_for_recovery = False
            while True:
                recovery = self._es.indices.recovery(self.index, params={"active_only": "true"})
                shards = recovery.get(self.index, {}).get("shards", [])
                if any(c["stage"] != "DONE" for c in shards):
                    waiting_for_recovery = True
                    s = 1000 / random.randint(1000, 3000)
                    time.sleep(s)
                    logger.warning("Elasticsearch index recovering")
                else:
                    if waiting_for_recovery:
                        logger.warning("Elasticsearch index recovery done")
                    break
            self.configured = True
            break
        else:
            raise Exception('Could not connect to server')

        # use_mapping_types
        if self.use_mapping_types is None:
            if self.version >= [7]:
                self.use_mapping_types = False
            else:
                mappings = set(list(self._es.indices.get_mapping(self.index).values())[0]['mappings'])
                self.use_mapping_types = self.LEGACY_DOC_TYPE not in mappings 

def __init__(self, es_hosts=None, es_port=None, es_ssl=False, es_auth=None,
                 verbose=False, country_threshold=0.6, threads=True,
                 progress=True, mod_date="2018-06-05", **kwargs):
        DATA_PATH = pkg_resources.resource_filename('mordecai', 'data/')
        MODELS_PATH = pkg_resources.resource_filename('mordecai', 'models/')
        self._cts = utilities.country_list_maker()
        self._just_cts = utilities.country_list_maker()
        self._inv_cts = utilities.make_inv_cts(self._cts)
        country_state_city = utilities.other_vectors()
        self._cts.update(country_state_city)
        self._ct_nlp = utilities.country_list_nlp(self._cts)
        self._prebuilt_vec = [w.vector for w in self._ct_nlp]
        self._both_codes = utilities.make_country_nationality_list(self._cts, DATA_PATH + "nat_df.csv")
        self._admin1_dict = utilities.read_in_admin1(DATA_PATH + "admin1CodesASCII.json")
        self.conn = utilities.setup_es(es_hosts, es_port, es_ssl, es_auth)
        self.country_model = keras.models.load_model(MODELS_PATH + "country_model.h5")
        self.rank_model = keras.models.load_model(MODELS_PATH + "rank_model.h5")
        self._skip_list = utilities.make_skip_list(self._cts)
        self.training_setting = False  # make this true if you want training formatted
        # if the best country guess is below the country threshold, don't return anything at all
        self.country_threshold = country_threshold
        feature_codes = pd.read_csv(DATA_PATH + "feature_codes.txt", sep="\t", header=None)
        self._code_to_text = dict(zip(feature_codes[1], feature_codes[3]))  # human readable geonames IDs
        self.verbose = verbose  # return the full dictionary or just the good parts?
        self.progress = progress  # display progress bars?
        self.threads = threads
        if 'n_threads' in kwargs.keys():
            warnings.warn("n_threads is deprecated. Use threads=True instead.", DeprecationWarning)
        try:
            # https://www.reddit.com/r/Python/comments/3a2erd/exception_catch_not_catching_everything/
            # with nostderr():
            self.conn.count()
        except:
            raise ConnectionError("""Could not establish contact with Elasticsearch at {0} on port {1}.
Are you sure it's running?
Mordecai needs access to the Geonames/Elasticsearch gazetteer to function.
See https://github.com/openeventdata/mordecai#installation-and-requirements
for instructions on setting up Geonames/Elasticsearch""".format(es_hosts, es_port))
        es_date = utilities.check_geonames_date(self.conn)
        if es_date != mod_date:
            print("""You may be using an outdated Geonames index.
Your index is from {0}, while the most recent is {1}. Please see
https://github.com/openeventdata/mordecai/ for instructions on updating.""".format(es_date, mod_date))