Python six.viewitems() Examples

def plot_avg_regret(policy):

    """Plot average regret with respect to time.

        Parameters
        ----------
        policy: bandit object
            The bandit algorithm you want to evaluate.
    """

    avg_reward = calculate_avg_reward(policy)
    points = sorted(six.viewitems(avg_reward), key=lambda x: x[0])
    x, y = zip(*points)
    plt.plot(x, [1 - reward for reward in y], 'r-', label="average regret")
    plt.xlabel('time')
    plt.ylabel('avg regret')
    plt.legend()
    plt.title("Average Regret with respect to Time") 

def form_valid(self, form):
        """
        Main functionality, creates :class:`Service` and calls
        :meth:`execute` with proper parameters.  If everything
        is successful, calls Base :meth:`form_valid`.  If error
        is throw, adds it to the form and calls :meth:`form_invalid`
        """
        try:
            cls = self.get_service_class()
            cls.execute(
                self.get_service_input(form),
                self.get_service_files(),
                **self.get_service_kwargs()
            )
            return super(ServiceViewMixin, self).form_valid(form)

        except InvalidInputsError as e:
            for k, v in viewitems(e.errors):
                form.add_error(k, v)
            return self.form_invalid(form)
        except ValidationError as e:
            form.add_error(None, e)
            return self.form_invalid(form) 

def dump(self):
        cls = self.__class__
        # FIXME: doesn't support recursion
        def stringify(layer, indent=0, tab='  '):
            data = (k for k, v in six.viewitems(layer) if not isinstance(v, node))
            result = []
            for k in data:
                result.append("{:s}{!r} -> {!r}".format(tab * indent, k, layer[k]))

            branches = [k for k, v in six.viewitems(layer) if isinstance(v, node)]
            for k in branches:
                result.append("{:s}{!r}".format(tab * indent, k))
                branch_data = stringify(layer[k], indent+1, tab=tab)
                result.extend(branch_data)
            return result
        return '\n'.join(("{!r}({:d})".format(cls, self.id), '\n'.join(stringify(self))))

### cache for looking up encoder/decoder types 

def match_node(self, key):
        found_node = None
        for regex_key, node in six.viewitems(self._key_node_dict):
            match_object = re.match("(?:%s)\Z" % regex_key, key)
            if match_object is not None:
                if found_node is None:
                    found_node = node
                    found_regex_key = regex_key
                    found_match_object = match_object
                else:
                    raise ValueError("The data key '{}' matches multiple keys: "
                                     "'{}' for {} and '{}' for {}.".format(
                                         key, found_regex_key, found_node,
                                         regex_key, node))
        if found_node is None:
            raise KeyError(key)
        return found_regex_key, found_node, found_match_object 

def get_data_keys_from_structure(structure):
    data_keys = []

    def _get_data_keys_from_structure(structure):
        if isinstance(structure, basestring):
            data_keys.append(structure)
        elif isinstance(structure, list):
            data_keys.extend(structure)
        elif isinstance(structure, dict):
            for _, val in six.viewitems(structure):
                _get_data_keys_from_structure(val)
        else:
            raise TypeError("The bundle structure only support "
                            "dict, list and str.")
    _get_data_keys_from_structure(structure)

    return data_keys 

def CaptureFrameLocals(self, frame):
    """Captures local variables and arguments of the specified frame.

    Args:
      frame: frame to capture locals and arguments.

    Returns:
      (arguments, locals) tuple.
    """
    # Capture all local variables (including method arguments).
    variables = {n: self.CaptureNamedVariable(n, v, 1,
                                              self.default_capture_limits)
                 for n, v in six.viewitems(frame.f_locals)}

    # Split between locals and arguments (keeping arguments in the right order).
    nargs = frame.f_code.co_argcount
    if frame.f_code.co_flags & inspect.CO_VARARGS: nargs += 1
    if frame.f_code.co_flags & inspect.CO_VARKEYWORDS: nargs += 1

    frame_arguments = []
    for argname in frame.f_code.co_varnames[:nargs]:
      if argname in variables: frame_arguments.append(variables.pop(argname))

    return (frame_arguments, list(six.viewvalues(variables))) 

def test_as_dict(self):
    conf_dict = conf._asdict()
    expected_dict = {
        'flag_key': 'flag_value',
        'other_flag': 'other_value',
        'true_value': True,
        'num_value': 100,
        'none_default': None,
        'string_default': 'default',
    }
    # assert first dict is a subset of second dict
    self.assertLessEqual(six.viewitems(expected_dict),
                              six.viewitems(conf_dict)) 

def inv_flat_tuple_dict(flat_tuple_dict):
    return {v: k for k, v in six.viewitems(flat_tuple_dict)} 

def get_flat_underscore_dict(flat_tuple_dict):
    return {'_'.join(k): v for k, v in six.viewitems(flat_tuple_dict)} 

def get_flat_path_dict(flat_tuple_dict):
    return {os.path.join(*k): v for k, v in six.viewitems(flat_tuple_dict)} 

def unflatten(d, splitter='tuple', inverse=False):
    """Unflatten dict-like object.

    Parameters
    ----------
    d : dict-like object
        The dict that will be unflattened.
    splitter : {'tuple', 'path', 'underscore', 'dot', Callable}
        The key splitting method. If a Callable is given, the Callable will be
        used to split `d`.
        'tuple': Use each element in the tuple key as the key of the unflattened dict.
        'path': Use `pathlib.Path.parts` to split keys.
        'underscore': Use underscores to split keys.
        'dot': Use underscores to split keys.
    inverse : bool
        Whether you want to invert the key and value before flattening.

    Returns
    -------
    unflattened_dict : dict
    """
    if isinstance(splitter, str):
        splitter = SPLITTER_DICT[splitter]

    unflattened_dict = {}
    for flat_key, value in six.viewitems(d):
        if inverse:
            flat_key, value = value, flat_key
        key_tuple = splitter(flat_key)
        nested_set_dict(unflattened_dict, key_tuple, value)

    return unflattened_dict 

def write_data(self, result_dict):
        for key, val in six.viewitems(result_dict):
            pickle_path = os.path.join(self.pickle_dir, key + ".pkl")
            with SimpleTimer("Writing generated data %s to pickle file" % key,
                             end_in_new_line=False), \
                    open(pickle_path, "wb") as fp:
                cPickle.dump(val, fp, protocol=cPickle.HIGHEST_PROTOCOL) 

def _grow_ancestors(self, nx_digraph, root_node_key, successor_keys,
                        re_args={}):
        successor_keys = {k: k.format(**re_args) for k in successor_keys}
        # grow the graph using DFS
        for template_key, key in six.viewitems(successor_keys):
            regex_key, node, match_object = self.match_node(key)

            # for merging node, we use key as the 'key' in nx_digraph
            mode = self._node_mode_dict[node]
            if mode == 'full':
                node_key = (self._node_key_dict[node]['keys']
                            + self._node_key_dict[node]['re_escape_keys'])
            elif mode == 'one':
                node_key = key
            else:
                raise ValueError("Mode '%s' is not supported." % mode)

            re_args = match_object.groupdict()
            if node_key not in nx_digraph:
                attr = self._node_attr_dict[node].copy()
                attr.setdefault('__name__', node)
                attr['__re_args__'] = re_args
                nx_digraph.add_node(node_key, attr)
                self._grow_ancestors(nx_digraph, node_key,
                                     self._node_succesor_dict[node], re_args)
            if not nx_digraph.has_edge(root_node_key, node_key):
                nx_digraph.add_edge(root_node_key, node_key,
                                    keys=set(), template_keys={})
            edge_attr = nx_digraph[root_node_key][node_key]
            edge_attr['keys'].add(key)
            edge_attr['template_keys'].update(((template_key, key),)) 

def bundle(self, structure, data_bundle_hdf_path, buffer_size=int(1e+9),
               structure_config=None):
        if structure_config is None:
            structure_config = {}

        def _bundle_data(structure, structure_config, dset_name=""):
            if isinstance(structure, basestring) and dset_name != "":
                (self.get_handler(structure)
                 .bundle(structure, data_bundle_hdf_path, dset_name))
            elif isinstance(structure, list):
                if structure_config.get('concat', False):
                    self.fill_concat_data(data_bundle_hdf_path, dset_name,
                                          structure, buffer_size)
                else:
                    for data_key in structure:
                        (self.get_handler(data_key)
                         .bundle(data_key, data_bundle_hdf_path,
                                 dset_name + "/" + data_key))
            elif isinstance(structure, dict):
                for key, val in six.viewitems(structure):
                    _bundle_data(val, structure_config.get(key, {}),
                                 dset_name + "/" + key)
            else:
                raise TypeError("The bundle structure only support "
                                "dict, list and str (except the first layer).")

        if os.path.isfile(data_bundle_hdf_path):
            os.remove(data_bundle_hdf_path)
        with SimpleTimer("Bundling data"):
            _bundle_data(structure, structure_config) 

def viewflatitems(self):
        ''' Return view of flattened items '''
        return six.viewitems(self.flattened()) 

def get_ids_to_read(self, *fields):
        """ Return list of ids, that have no at least one of specified
            fields in cache

            For example::

                cache.get_ids_to_read('name', 'country_id', 'parent_id')

            This code will traverse all record ids managed by this cache,
            and find those that have no at least one field in cache.
            This is highly useful in prefetching
        """
        return [key for key, val in six.viewitems(self)
                if any(((field not in val) for field in fields))] 

def contents(Contents, **tagmap):
        '''Apply the tags in `Contents` back into each function within the database.'''
        global apply
        cls, tagmap_output = apply.__class__, u", {:s}".format(u', '.join(u"{:s}={:s}".format(internal.utils.string.escape(oldtag), internal.utils.string.escape(newtag)) for oldtag, newtag in six.iteritems(tagmap))) if tagmap else ''

        count = 0
        for loc, res in Contents:
            ea = locationToAddress(loc)

            # warn the user if this address is not within a function
            if not func.within(ea):
                logging.warn(u"{:s}.contents(...{:s}) : Address {:#x} is not within a function. Using a global tag.".format('.'.join((__name__, cls.__name__)), tagmap_output, ea))

            # grab the current (old) tag state
            state = db.tag(ea)

            # transform the new tag state using the tagmap
            new = { tagmap.get(name, name) : value for name, value in six.viewitems(res) }

            # check if the tag mapping resulted in the deletion of a tag
            if len(new) != len(res):
                for name in six.viewkeys(res) - six.viewkeys(new):
                    logging.warn(u"{:s}.contents(...{:s}) : Refusing requested tag mapping as it results in the tag \"{:s}\" overwriting tag \"{:s}\" for the contents at {:#x}. The value {!s} would be overwritten by {!s}.".format('.'.join((__name__, cls.__name__)), tagmap_output, internal.utils.string.escape(name, '"'), internal.utils.string.escape(tagmap[name], '"'), ea, internal.utils.string.repr(res[name]), internal.utils.string.repr(res[tagmap[name]])))
                pass

            # inform the user if any tags are being overwritten with different values
            for name in six.viewkeys(state) & six.viewkeys(new):
                if state[name] == new[name]: continue
                logging.warn(u"{:s}.contents(...{:s}) : Overwriting contents tag \"{:s}\" for address {:#x} with new value {!s}. Old value was {!s}.".format('.'.join((__name__, cls.__name__)), tagmap_output, internal.utils.string.escape(name, '"'), ea, internal.utils.string.repr(new[name]), internal.utils.string.repr(state[name])))

            # write the tags to the contents address
            try:
                [ db.tag(ea, name, value) for name, value in six.iteritems(new) if state.get(name, dummy) != value ]
            except:
                logging.warn(u"{:s}.contents(...{:s}) : Unable to apply tags {!s} to location {:#x}.".format('.'.join((__name__, cls.__name__)), tagmap_output, internal.utils.string.repr(new), ea), exc_info=True)

            # increase our counter
            count += 1
        return count

    ## applying frames to all the functions 

def globals(Globals, **tagmap):
        '''Apply the tags in `Globals` back into the database.'''
        global apply
        cls, tagmap_output = apply.__class__, u", {:s}".format(u', '.join(u"{:s}={:s}".format(internal.utils.string.escape(oldtag), internal.utils.string.escape(newtag)) for oldtag, newtag in six.iteritems(tagmap))) if tagmap else ''

        count = 0
        for ea, res in Globals:
            ns = func if func.within(ea) else db

            # grab the current (old) tag state
            state = ns.tag(ea)

            # transform the new tag state using the tagmap
            new = { tagmap.get(name, name) : value for name, value in six.viewitems(res) }

            # check if the tag mapping resulted in the deletion of a tag
            if len(new) != len(res):
                for name in six.viewkeys(res) - six.viewkeys(new):
                    logging.warn(u"{:s}.globals(...{:s}) : Refusing requested tag mapping as it results in the tag \"{:s}\" overwriting the tag \"{:s}\" in the global {:#x}. The value {!s} would be replaced with {!s}.".format('.'.join((__name__, cls.__name__)), tagmap_output, internal.utils.string.escape(name, '"'), internal.utils.string.escape(tagmap[name], '"'), ea, internal.utils.string.repr(res[name]), internal.utils.string.repr(res[tagmap[name]])))
                pass

            # check what's going to be overwritten with different values prior to doing it
            for name in six.viewkeys(state) & six.viewkeys(new):
                if state[name] == new[name]: continue
                logging.warn(u"{:s}.globals(...{:s}) : Overwriting tag \"{:s}\" for global at {:#x} with new value {!s}. Old value was {!s}.".format('.'.join((__name__, cls.__name__)), tagmap_output, internal.utils.string.escape(name, '"'), ea, internal.utils.string.repr(new[name]), internal.utils.string.repr(state[name])))

            # now we can apply the tags to the global address
            try:
                [ ns.tag(ea, name, value) for name, value in six.iteritems(new) if state.get(name, dummy) != value ]
            except:
                logging.warn(u"{:s}.globals(...{:s}) : Unable to apply tags ({!s}) to global {:#x}.".format('.'.join((__name__, cls.__name__)), tagmap_output, internal.utils.string.repr(new), ea), exc_info=True)

            # increase our counter
            count += 1
        return count

    ## applying contents tags to all the functions 

def hashable_lru(maxsize=16):
    def hashable_cache_internal(func):
        cache = lru_cache(maxsize=maxsize)
        def deserialize(value):
            if isinstance(value, Serialized):
                return json.loads(value.json)
            else:
                return value

        def func_with_serialized_params(*args, **kwargs):
            _args = tuple([deserialize(arg) for arg in args])
            _kwargs = {k: deserialize(v) for k, v in six.viewitems(kwargs)}
            return func(*_args, **_kwargs)

        cached_func = cache(func_with_serialized_params)

        @wraps(func)
        def hashable_cached_func(*args, **kwargs):
            _args = tuple([
                Serialized(json.dumps(arg, sort_keys=True))
                if type(arg) in (list, dict) else arg
                for arg in args
            ])
            _kwargs = {
                k: Serialized(json.dumps(v, sort_keys=True))
                if type(v) in (list, dict) else v
                for k, v in kwargs.items()
            }
            return copy.deepcopy(cached_func(*_args, **_kwargs))
        hashable_cached_func.cache_info = cached_func.cache_info
        hashable_cached_func.cache_clear = cached_func.cache_clear
        return hashable_cached_func

    return hashable_cache_internal 

def get_format_str(data):
    for format_str, format_class in six.viewitems(FORMAT_DICT):
        if isinstance(data, format_class):
            return format_str
    raise ValueError("Data type {} is not supported.".format(type(data))) 

def reward(self, history_id, rewards):
        """Reward the previous action with reward.

        Parameters
        ----------
        history_id : int
            The history id of the action to reward.

        rewards : dictionary
            The dictionary {action_id, reward}, where reward is a float.
        """
        context = (self._historystorage
                   .get_unrewarded_history(history_id)
                   .context)

        model = self._modelstorage.get_model()
        w = model['w']
        action_probs = model['action_probs']
        action_ids = list(six.viewkeys(six.next(six.itervalues(context))))

        # Update the model
        for action_id, reward in six.viewitems(rewards):
            y_hat = {}
            v_hat = {}
            for i in six.viewkeys(context):
                y_hat[i] = (context[i][action_id] * reward
                            / action_probs[action_id])
                v_hat[i] = sum(
                    [context[i][k] / action_probs[k] for k in action_ids])
                w[i] = w[i] * np.exp(
                    self.p_min / 2
                    * (y_hat[i] + v_hat[i]
                       * np.sqrt(np.log(len(context) / self.delta)
                                 / (len(action_ids) * self.max_rounds))))

        self._modelstorage.save_model({
            'action_probs': action_probs, 'w': w})

        # Update the history
        self._historystorage.add_reward(history_id, rewards) 

def reward(self, history_id, rewards):
        """Reward the previous action with reward.

        Parameters
        ----------
        history_id : int
            The history id of the action to reward.

        rewards : dictionary
            The dictionary {action_id, reward}, where reward is a float.
        """
        context = (self._history_storage
                   .get_unrewarded_history(history_id)
                   .context)

        # Update the model
        model = self._model_storage.get_model()
        A = model['A']  # pylint: disable=invalid-name
        A_inv = model['A_inv']  # pylint: disable=invalid-name
        b = model['b']
        theta = model['theta']

        for action_id, reward in six.viewitems(rewards):
            action_context = np.reshape(context[action_id], (-1, 1))
            A[action_id] += action_context.dot(action_context.T)
            A_inv[action_id] = np.linalg.inv(A[action_id])
            b[action_id] += reward * action_context
            theta[action_id] = A_inv[action_id].dot(b[action_id])
        self._model_storage.save_model({
            'A': A,
            'A_inv': A_inv,
            'b': b,
            'theta': theta,
        })

        # Update the history
        self._history_storage.add_reward(history_id, rewards) 

def reward(self, history_id, rewards):
        """Reward the previous action with reward.

        Parameters
        ----------
        history_id : int
            The history id of the action to reward.

        rewards : dictionary
            The dictionary {action_id, reward}, where reward is a float.
        """
        context = (self._history_storage
                   .get_unrewarded_history(history_id)
                   .context)

        # Update the model
        model = self._model_storage.get_model()
        B = model['B']  # pylint: disable=invalid-name
        f = model['f']

        for action_id, reward in six.viewitems(rewards):
            context_t = np.reshape(context[action_id], (-1, 1))
            B += context_t.dot(context_t.T)  # pylint: disable=invalid-name
            f += reward * context_t
            mu_hat = np.linalg.inv(B).dot(f)
        self._model_storage.save_model({'B': B, 'mu_hat': mu_hat, 'f': f})

        # Update the history
        self._history_storage.add_reward(history_id, rewards) 

def get_advice(context, action_ids, experts):
    advice = {}
    for t, context_t in six.viewitems(context):
        advice[t] = {}
        for exp_i, expert in enumerate(experts):
            prob = expert.predict_proba(context_t[np.newaxis, :])[0]
            advice[t][exp_i] = {}
            for action_id, action_prob in zip(action_ids, prob):
                advice[t][exp_i][action_id] = action_prob
    return advice 

def push(self, expunge=False):
        """Push up cache data to Zookeeper.

        :param ``bool`` expunge:
            If `True`, remove all remote files not present locally.
        :returns:
            `True` - When new data was pushed up to Zookeeper.
            `False` - When Zookeeper was already up to date.
        """
        assert self.zkclient is not None, 'Operation requires a ZK client.'

        new_data = False
        for name, entries in six.iteritems(self._cached):
            # We only consider the latest version (in case of dirty cache).
            cache_entry = entries[0]
            zk_entry = (
                self._zkdata[name][0]
                if name in self._zkdata
                else None
            )
            # Upload a new version if the chksum is different.
            if zk_entry is None or cache_entry.chksum != zk_entry.chksum:
                with io.open(cache_entry.fname, 'rb') as data:
                    self.zkclient.create(
                        '{path}/{name}#{chksum}#'.format(
                            path=self._zkpath,
                            name=name,
                            chksum=cache_entry.chksum
                        ),
                        data.read(),
                        sequence=True,
                        makepath=True
                    )
                new_data = True

        if new_data:
            self.refresh_zk()

        for name, entries in list(six.viewitems(self._zkdata)):
            if name not in self._cached:
                if expunge:
                    # Clean up all sequence numbers for that name.
                    for zk_entry in entries:
                        zkutils.ensure_deleted(self.zkclient, zk_entry.zname)
            else:
                # Clean up all "older" sequence numbers for that name.
                for zk_entry in entries[1:]:
                    zkutils.ensure_deleted(self.zkclient, zk_entry.zname)

        return new_data 

def synchronize(self):
        """Cleanup state resource.
        """
        for app_unique_name in six.viewkeys(self._devices.copy()):
            if not self._devices[app_unique_name].get('stale', False):
                continue

            # This is a stale device, destroy it.
            self.on_delete_request(app_unique_name)

        # Reset the container environment sets to the IP we have now cleaned
        # up.  This is more complex than expected because multiple environment
        # can be merged in the same set in _SET_BY_ENVIRONMENT.
        container_env_ips = {}
        for set_name in set(_SET_BY_ENVIRONMENT.values()):
            key = sorted(
                [
                    env for env in _SET_BY_ENVIRONMENT
                    if _SET_BY_ENVIRONMENT[env] == set_name
                ]
            )
            container_env_ips[tuple(key)] = set()

        for set_envs, set_ips in six.viewitems(container_env_ips):
            for device in six.viewvalues(self._devices):
                if device['environment'] not in set_envs:
                    continue
                set_ips.add(device['ip'])

        for set_envs, set_ips in six.viewitems(container_env_ips):
            iptables.atomic_set(
                _SET_BY_ENVIRONMENT[set_envs[0]],
                set_ips,
                set_type='hash:ip',
                family='inet', hashsize=1024, maxelem=65536
            )

        # It is now safe to clean up all remaining vIPs without resource.
        self._vips.garbage_collect()

        # Read bridge status
        self._bridge_mtu = netdev.dev_mtu(self._TMBR_DEV)