Python kazoo.exceptions.NodeExistsError() Examples

def set_or_create(self, key, value, version=-1):

        value = self._dump(value)
        while True:
            try:
                self.zkclient.set(self.get_path(key), value, version=version)
                return
            except NoNodeError:
                if version == -1:
                    try:
                        self.zkclient.create(self.get_path(key), value, acl=self._get_acl())
                        return
                    except NodeExistsError:
                        continue
                else:
                    raise 

def _create(self):

        logger.debug('to creaet: {s}'.format(s=str(self)))

        try:
            self.zkclient.create(self.lock_path,
                                 utfjson.dump(self.identifier),
                                 ephemeral=self.ephemeral,
                                 acl=self.zkconf.kazoo_digest_acl())

        except NodeExistsError as e:

            # NOTE Success create on server side might also results in failure
            # on client side due to network issue.
            # 'get' after 'create' to check if existent node belongs to this
            # client.

            logger.debug(repr(e) + ' while create lock: {s}'.format(s=str(self)))
            self.lock_holder = None
            return

        logger.info('CREATE OK: {s}'.format(s=str(self))) 

def update_task(self, task_id: str, **kwargs):
        retry = True
        while retry:
            retry = False
            existing_task, stat = self._get_task(task_id)

            zk_path = self._zk_path_from_task_id(task_id)
            if existing_task:
                merged_params = existing_task.merge(**kwargs)
                try:
                    self.zk_client.set(
                        zk_path, merged_params.serialize(), version=stat.version
                    )
                except BadVersionError:
                    retry = True
            else:
                merged_params = MesosTaskParameters(**kwargs)
                try:
                    self.zk_client.create(zk_path, merged_params.serialize())
                except NodeExistsError:
                    retry = True

        return merged_params 

def reallocate_partitions(self, partitions_data: list) -> bool:
        j = {
            "version": "1",
            "partitions": [
                {
                    "topic": topic,
                    "partition": int(partition),
                    "replicas": [int(p) for p in replicas]
                } for (topic, partition, replicas) in partitions_data]
        }
        try:
            data = json.dumps(j)
            self.exhibitor.create("/admin/reassign_partitions", data.encode('utf-8'))
            _LOG.info("Reallocating {}".format(data))
            return True
        except NodeExistsError:
            _LOG.info("Waiting for free reallocation slot, still in progress...")
        return False 

def test_add_service_instance_with_concurrency_request_error(
        test_application_name, add_service, mocker):
    key = '169.254.1.2_5000'
    value = '{"ip": "169.254.1.2", "port":{"main": 5000},"state":"up"}'
    mocker.patch.object(
        service_client.raw_client, 'create', side_effect=NodeExistsError)

    _, r = add_service(key, value)
    assert r.status_code == 409
    assert r.json['status'] == 'Conflict'
    assert r.json['message'] == 'resource is modified by another request'

    mocker.patch.object(service_client.raw_client, 'create', return_value=None)
    mocker.patch.object(service_client.raw_client, 'exists', return_value=None)

    _, r = add_service(key, value)
    assert r.status_code == 409
    assert r.json['status'] == 'Conflict'
    assert r.json['message'] == 'resource is modified by another request' 

def _exc_wrapper(self):
        """Exception context-manager which wraps kazoo exceptions.

        This is used to capture and wrap any kazoo specific exceptions and
        then group them into corresponding taskflow exceptions (not doing
        that would expose the underlying kazoo exception model).
        """
        try:
            yield
        except self._client.handler.timeout_exception:
            exc.raise_with_cause(exc.StorageFailure,
                                 "Storage backend timeout")
        except k_exc.SessionExpiredError:
            exc.raise_with_cause(exc.StorageFailure,
                                 "Storage backend session has expired")
        except k_exc.NoNodeError:
            exc.raise_with_cause(exc.NotFound,
                                 "Storage backend node not found")
        except k_exc.NodeExistsError:
            exc.raise_with_cause(exc.Duplicate,
                                 "Storage backend duplicate node")
        except (k_exc.KazooException, k_exc.ZookeeperError):
            exc.raise_with_cause(exc.StorageFailure,
                                 "Storage backend internal error") 

def process_end(self):
        try:
            self._zk.create(self._path+"/"+self._name+"complete")
        except NodeExistsError:
            pass 

def process_start(self):
        if self.processed(): return False
        try:
            self._zk.create(self._path+"/"+self._name+"processing",ephemeral=True)
            return True
        except NodeExistsError: # another process wins
            return False 

def set(self, path, value):
        value=json.dumps(value).encode('utf-8')
        try:
            self._zk.create(path, value, makepath=True)
        except NodeExistsError:
            self._zk.set(path,value) 

def initial(self, data):

        #
        # - the /hash node is where we store the md5 hash of all our pods + their dependencies
        # - the /snapshot node is where we store the last known state of our pods (e.g where they run from and what
        #   their port mapping is)
        #
        try:
            self.zk.create('%s/%s.%s/snapshot' % (ROOT, self.scope, self.tag), value='{}', ephemeral=True)

        except NodeExistsError:
            pass

        #
        # - start the watch our local pods
        # - this ancillary actor will piggy-back on our zk client and use it to query our pod
        #   information on a regular basis
        #
        data.dirty = 0
        data.last = None
        data.next_probe = 0
        self.snapshots['local'] = {}
        self.watchers = [Local.start(self.actor_ref, self.zk, self.scope, self.tag)]

        #
        # - add a set of extra watchers for our dependencies
        # - make sure to look for clusters within our own namespace
        # - start spinning (the watcher updates will be processed in there)
        #
        self.watchers += [Remote.start(self.actor_ref, self.zk, self.scope, self.tag, tag) for tag in self.depends_on]
        logger.debug('%s : watching %d dependencies' % (self.path, len(self.depends_on)))
        logger.info('%s : leading for cluster %s.%s' % (self.path, self.scope, self.tag))
        return 'spin', data, 0 

def add_job(self, job):
        self._ensure_paths()
        node_path = os.path.join(self.path,  str(job.id))
        value = {
            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
            'job_state': job.__getstate__()
        }
        data = pickle.dumps(value, self.pickle_protocol)
        try:
            self.client.create(node_path, value=data)
        except NodeExistsError:
            raise ConflictingIdError(job.id) 

def add_job(self, job):
        self._ensure_paths()
        node_path = os.path.join(self.path,  str(job.id))
        value = {
            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
            'job_state': job.__getstate__()
        }
        data = pickle.dumps(value, self.pickle_protocol)
        try:
            self.client.create(node_path, value=data)
        except NodeExistsError:
            raise ConflictingIdError(job.id) 

def touch_member(self, data, permanent=False):
        cluster = self.cluster
        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
        encoded_data = json.dumps(data, separators=(',', ':')).encode('utf-8')
        if member and (self._client.client_id is not None and member.session != self._client.client_id[0] or
                       not (deep_compare(member.data.get('tags', {}), data.get('tags', {})) and
                            member.data.get('version') == data.get('version') and
                            member.data.get('checkpoint_after_promote') == data.get('checkpoint_after_promote'))):
            try:
                self._client.delete_async(self.member_path).get(timeout=1)
            except NoNodeError:
                pass
            except Exception:
                return False
            member = None

        if member:
            if deep_compare(data, member.data):
                return True
        else:
            try:
                self._client.create_async(self.member_path, encoded_data, makepath=True,
                                          ephemeral=not permanent).get(timeout=1)
                return True
            except Exception as e:
                if not isinstance(e, NodeExistsError):
                    logger.exception('touch_member')
                    return False
        try:
            self._client.set_async(self.member_path, encoded_data).get(timeout=1)
            return True
        except Exception:
            logger.exception('touch_member')

        return False 

def register_entity(self, entity):
        entity_type = entity.kind
        if entity_type == c_base.Conductor.ENTITY_KIND:
            entity_path = k_paths.join(self.entity_path, entity_type)
            try:
                self._client.ensure_path(entity_path)
                self._client.create(k_paths.join(entity_path, entity.name),
                                    value=misc.binary_encode(
                                        jsonutils.dumps(entity.to_dict())),
                                    ephemeral=True)
            except k_exceptions.NodeExistsError:
                pass
            except self._client.handler.timeout_exception:
                excp.raise_with_cause(
                    excp.JobFailure,
                    "Can not register entity %s under %s, operation"
                    " timed out" % (entity.name, entity_path))
            except k_exceptions.SessionExpiredError:
                excp.raise_with_cause(
                    excp.JobFailure,
                    "Can not register entity %s under %s, session"
                    " expired" % (entity.name, entity_path))
            except k_exceptions.KazooException:
                excp.raise_with_cause(
                    excp.JobFailure,
                    "Can not register entity %s under %s, internal"
                    " error" % (entity.name, entity_path))
        else:
            raise excp.NotImplementedError(
                "Not implemented for other entity type '%s'" % entity_type) 

def create_offsets(zk, consumer_group, offsets):
    """Create path with offset value for each topic-partition of given consumer
    group.

    :param zk: Zookeeper client
    :param consumer_group: Consumer group id for given offsets
    :type consumer_group: int
    :param offsets: Offsets of all topic-partitions
    :type offsets: dict(topic, dict(partition, offset))
    """
    # Create new offsets
    for topic, partition_offsets in six.iteritems(offsets):
        for partition, offset in six.iteritems(partition_offsets):
            new_path = "/consumers/{groupid}/offsets/{topic}/{partition}".format(
                groupid=consumer_group,
                topic=topic,
                partition=partition,
            )
            try:
                zk.create(new_path, value=offset, makepath=True)
            except NodeExistsError:
                print(
                    "Error: Path {path} already exists. Please re-run the "
                    "command.".format(path=new_path),
                    file=sys.stderr,
                )
                raise 

def test_znode_model_create_oos(zk, faker, mocker, schema, model_class):
    name = faker.uuid4()
    model = model_class(zk, application_name=name)
    model.data = '{}'

    zk.create('/huskar/service/%s/overall' % name, b'1s', makepath=True)
    with raises(OutOfSyncError) as error:
        model.save()
    assert isinstance(error.value.args[0], NodeExistsError)
    assert model.stat is None
    assert model.data == '{}' 

def save(self, version=None):
        """Saves the data in this instance to ZooKeeper.

        It is concurrency-safe if you never break the :attr:`ZnodeModel.stat`.

        :param version: Optional. The alternative version instead of
                        :attr:`ZnodeModel.stat`.
        :raises OutOfSyncError: The local data is outdated.
        :raises marshmallow.ValidationError: :attr:`ZnodeModel.data` is invalid
        """
        data, _ = self.MARSHMALLOW_SCHEMA.dumps(self.data)
        self.MARSHMALLOW_SCHEMA.loads(data)  # raise ValidationError if need
        if self.stat is None:
            try:
                self.client.create(self.path, data, makepath=True)
                zk_payload(payload_data=data, payload_type='create')
            except NodeExistsError as e:
                raise OutOfSyncError(e)
            self.load()
        else:
            if version is None:
                version = self.stat.version
            try:
                self.stat = self.client.set(self.path, data, version=version)
                zk_payload(payload_data=data, payload_type='set')
            except (NoNodeError, BadVersionError) as e:
                raise OutOfSyncError(e) 

def create_if_not_exist(self, application, cluster=None, strict=False):
        path = self.get_path(application, cluster)
        try:
            self.raw_client.create(path, b'', makepath=True)
            zk_payload(payload_data=b'', payload_type='create')
        except NodeExistsError:
            if strict:
                target = 'application' if cluster is None else 'cluster'
                raise DataExistsError('%s exists already' % target) 

def test_reallocate_partition():
    call_idx = [0]

    def _create(path, value=None, **kwargs):
        if path in ('/bubuku/changes', '/bubuku/actions/global'):
            pass
        elif path == '/admin/reassign_partitions':
            if call_idx[0] >= 5:
                raise NodeExistsError()
            call_idx[0] += 1
            j = json.loads(value.decode('utf-8'))
            assert j['version'] == '1'
            assert len(j['partitions']) == 1
            p = j['partitions'][0]
            assert p['topic'] == 't01'
            assert p['partition'] == 0
            assert p['replicas'] == [1, 2, 3]
        else:
            raise NotImplementedError('Not implemented for path {}'.format(path))

    exhibitor_mock = MagicMock()
    exhibitor_mock.create = _create

    buku = BukuExhibitor(exhibitor_mock)

    assert buku.reallocate_partition('t01', 0, ['1', '2', '3'])
    assert buku.reallocate_partition('t01', 0, ['1', '2', 3])
    assert buku.reallocate_partition('t01', 0, [1, 2, 3])
    assert buku.reallocate_partition('t01', 0, [1, 2, 3])
    assert buku.reallocate_partition('t01', 0, [1, 2, 3])
    # Node exists
    assert not buku.reallocate_partition('t01', 0, [1, 2, 3]) 

def register_action(self, data: dict, broker_id: str = 'global'):
        registered = False
        while not registered:
            name = '/bubuku/actions/{}/{}'.format(broker_id, uuid.uuid4())
            try:
                self.exhibitor.create(name, json.dumps(data).encode('utf-8'), makepath=True)
                _LOG.info('Action {} registered with name {}'.format(data, name))
                registered = True
            except NodeExistsError:
                pass 

def update_disk_stats(self, broker_id: str, data: dict):
        data_bytes = json.dumps(data, separators=(',', ':')).encode('utf-8')
        path = '/bubuku/size_stats/{}'.format(broker_id)
        try:
            self.exhibitor.create(path, data_bytes, ephemeral=True, makepath=True)
        except NodeExistsError:
            self.exhibitor.set(path, data_bytes) 

def _lock_and_get_entry(self, entry_node: str) -> Optional[Tuple[bytes, ZnodeStat]]:
        try:
            lock_path = f"{self.locks_path}/{entry_node}"
            self.locked_entry_nodes.add(entry_node)
            self.client.create(lock_path, value=self.id, ephemeral=True)
        except NodeExistsError:
            self.locked_entry_nodes.add(entry_node)
            return None

        try:
            return self.client.get(f"{self.entries_path}/{entry_node}")
        except NoNodeError:
            self.client.delete(lock_path)
            return None 

def start_deployd(context):
    try:
        os.makedirs("/nail/etc/services")
    except OSError as e:
        if e.errno == errno.EEXIST:
            pass
    with ZookeeperPool() as zk:
        try:
            zk.create("/autoscaling")
        except NodeExistsError:
            pass
    context.zk_hosts = "%s/mesos-testcluster" % get_service_connection_string(
        "zookeeper"
    )
    context.soa_dir = "/nail/etc/services"
    if not hasattr(context, "daemon"):
        context.daemon = Popen("paasta-deployd", stderr=PIPE)
    output = context.daemon.stderr.readline().decode("utf-8")
    start = time.time()
    timeout = start + 60
    while "Startup finished!" not in output:
        output = context.daemon.stderr.readline().decode("utf-8")
        if not output:
            raise Exception("deployd exited prematurely")
        print(output.rstrip("\n"))
        if time.time() > timeout:
            raise Exception("deployd never ran")

    context.num_workers_crashed = 0

    def dont_let_stderr_buffer():
        while True:
            line = context.daemon.stderr.readline()
            if not line:
                return
            if DEAD_DEPLOYD_WORKER_MESSAGE.encode("utf-8") in line:
                context.num_workers_crashed += 1
            print(f"deployd stderr: {line}")

    threading.Thread(target=dont_let_stderr_buffer).start()
    time.sleep(5) 

def execute_plan(self, plan, allow_rf_change=False, allow_rf_mismatch=False):
        """Submit reassignment plan for execution."""
        reassignment_path = '{admin}/{reassignment_node}'\
            .format(admin=ADMIN_PATH, reassignment_node=REASSIGNMENT_NODE)
        plan_json = dump_json(plan)
        topic_names_from_proposed_plan = set()
        for partition in plan['partitions']:
            topic_names_from_proposed_plan.add(partition['topic'])
        base_plan = self.get_cluster_plan(topic_names=list(topic_names_from_proposed_plan))
        if not validate_plan(plan, base_plan, allow_rf_change=allow_rf_change, allow_rf_mismatch=allow_rf_mismatch):
            _log.error('Given plan is invalid. Aborting new reassignment plan ... {plan}'.format(plan=plan))
            return False
        # Send proposed-plan to zookeeper
        try:
            _log.info('Sending plan to Zookeeper...')
            self.create(reassignment_path, plan_json, makepath=True)
            _log.info(
                'Re-assign partitions node in Zookeeper updated successfully '
                'with {plan}'.format(plan=plan),
            )
            return True
        except NodeExistsError:
            _log.warning('Previous plan in progress. Exiting..')
            _log.warning('Aborting new reassignment plan... {plan}'.format(plan=plan))
            in_progress_plan = load_json(self.get(reassignment_path)[0])
            in_progress_partitions = [
                '{topic}-{p_id}'.format(
                    topic=p_data['topic'],
                    p_id=str(p_data['partition']),
                )
                for p_data in in_progress_plan['partitions']
            ]
            _log.warning(
                '{count} partition(s) reassignment currently in progress:-'
                .format(count=len(in_progress_partitions)),
            )
            _log.warning(
                '{partitions}. In Progress reassignment plan...'.format(
                    partitions=', '.join(in_progress_partitions),
                ),
            )
            return False
        except Exception as e:
            _log.error(
                'Could not re-assign partitions {plan}. Error: {e}'
                .format(plan=plan, e=e),
            )
            return False 

def claim(self, job, who):
        def _unclaimable_try_find_owner(cause):
            try:
                owner = self.find_owner(job)
            except Exception:
                owner = None
            if owner:
                message = "Job %s already claimed by '%s'" % (job.uuid, owner)
            else:
                message = "Job %s already claimed" % (job.uuid)
            excp.raise_with_cause(excp.UnclaimableJob,
                                  message, cause=cause)

        with self._wrap(job.uuid, job.path,
                        fail_msg_tpl="Claiming failure: %s"):
            # NOTE(harlowja): post as json which will allow for future changes
            # more easily than a raw string/text.
            value = jsonutils.dumps({
                'owner': who,
            })
            # Ensure the target job is still existent (at the right version).
            job_data, job_stat = self._client.get(job.path)
            txn = self._client.transaction()
            # This will abort (and not create the lock) if the job has been
            # removed (somehow...) or updated by someone else to a different
            # version...
            txn.check(job.path, version=job_stat.version)
            txn.create(job.lock_path, value=misc.binary_encode(value),
                       ephemeral=True)
            try:
                kazoo_utils.checked_commit(txn)
            except k_exceptions.NodeExistsError as e:
                _unclaimable_try_find_owner(e)
            except kazoo_utils.KazooTransactionException as e:
                if len(e.failures) < 2:
                    raise
                else:
                    if isinstance(e.failures[0], k_exceptions.NoNodeError):
                        excp.raise_with_cause(
                            excp.NotFound,
                            "Job %s not found to be claimed" % job.uuid,
                            cause=e.failures[0])
                    if isinstance(e.failures[1], k_exceptions.NodeExistsError):
                        _unclaimable_try_find_owner(e.failures[1])
                    else:
                        excp.raise_with_cause(
                            excp.UnclaimableJob,
                            "Job %s claim failed due to transaction"
                            " not succeeding" % (job.uuid), cause=e) 

def save(cls, application, cluster, key, value=None, runtime=None,
             version=None):
        """Register a service instance.

        :param str application: The name of application (appid).
        :param str cluster: The name of cluster.
        :param str key: The fingerprint of service instance.
        :param dict value: The information of service instance.
        :param dict runtime: The overlay information of service instance.
        :param int version: The version of service instance.
        """
        cls.check_cluster_name_in_creation(application, cluster)
        cluster_path = cls.client.get_path(application, cluster)
        service_path = cls.client.get_path(
            application, cluster, encode_key(key))

        raw_client = cls.client.raw_client
        raw_client.ensure_path(cluster_path)

        merged_value = {}
        merged_value.update(value or {})
        merged_value.update(runtime or {})

        try:
            remote_value, stat = raw_client.get(service_path)
        except NoNodeError:
            if not value:
                raise ServiceValueError(
                    '`value` should be provided while creating service.')
            json_merged_value = json.dumps(merged_value)
            try:
                raw_client.create(service_path, json_merged_value)
                zk_payload(payload_data=json_merged_value,
                           payload_type='create')
            except NodeExistsError as e:
                raise OutOfSyncError(e)
            stat = raw_client.exists(service_path)
            if stat is None:
                raise OutOfSyncError()
            return cls.info_class(merged_value, stat)
        else:
            if version is None:
                version = stat.version
            try:
                remote_value = json.loads(remote_value)
            except (ValueError, TypeError):
                logger.warning('Failed to parse %r', service_path)
                remote_value = {}
            new_value = dict(remote_value)
            new_value.update(value or {})
            new_value.update(runtime or {})
            json_new_value = json.dumps(new_value)
            try:
                stat = raw_client.set(
                    service_path, json_new_value, version=version)
                zk_payload(payload_data=json_new_value, payload_type='set')
            except BadVersionError as e:
                raise OutOfSyncError(e)
            return cls.info_class(new_value, stat) 

def wait_for_cnx(self, data):

        if self.force_reset or self.terminate:
            raise Aborted('resetting')

        #
        # - loop back if we haven't received a CONNECTED event from the driver
        #
        if not self.connected:
            return 'wait_for_cnx', data, SAMPLING

        #
        # - the /pods node holds all our ephemeral per-container data (one container == one child node)
        # - the /hash node stores the last recorded md5 hash (local pods + dependencies), which we use to
        #   flag any change amongst the pods or their dependencies
        #
        data.zk.ensure_path('%s/pods' % self.prefix)
        data.zk.ensure_path('%s/hash' % self.prefix)
        try:

            #
            # - register ourselves by creating an ephemeral
            # - this is where we can store arbitrary information (e.g our breadcrumbs)
            # - we ask for a sequence counter as well which we then keep (e.g in case of connection loss or reset
            #   we guarantee the pod won't get assigned a new index)
            # - this is *critical* for some use-cases (e.g Kafka where the broker index must remain the same)
            #
            path = data.zk.create('%s/pods/%s.' % (self.prefix, self.id), ephemeral=True, sequence=True)
            tokens = path.split('.')
            if self.seq is None:
                self.seq = int(tokens[-1])
            self.breadcrumbs['seq'] = self.seq
            js = json.dumps(self.breadcrumbs)
            data.zk.set(path, js)

        except NodeExistsError:

            #
            # - if the node is already there we just recovered from a zookeeper connection loss
            #   and /snapshot has not been phased out yet .. this is not an issue, simply pause a bit
            #   to re-attempt later
            #
            logger.debug('%s : pod %s is already there (probably a zk reconnect)' % (self.path, self.id))
            return 'wait_for_cnx', data, 5.0 * SAMPLING

        logger.debug('%s : registered as %s (#%d)' % (self.path, self.id, self.seq))
        data.connected_at = time.time()
        return 'spin', data, 0