Python dotmap.DotMap() Examples

def read_cfg(config_filename='configs/main.cfg', verbose=False):
    parser = ConfigParser()
    parser.optionxform = str
    parser.read(config_filename)
    cfg = DotMap()

    if verbose:
        hyphens = '-' * int((80 - len(config_filename))/2)
        print(hyphens + ' ' + config_filename + ' ' + hyphens)

    for section_name in parser.sections():
        if verbose:
            print('[' + section_name + ']')
        for name, value in parser.items(section_name):
            value = ConvertIfStringIsInt(value)
            cfg[name] = value
            spaces = ' ' * (30 - len(name))
            if verbose:
                print(name + ':' + spaces + str(cfg[name]))

    return cfg 

def load_current_kube_credentials():
    with open(os.path.expanduser(KUBE_CONFIG_PATH), 'r') as f:
        config = DotMap(yaml.safe_load(f))

        ctx_name = config['current-context']
        ctx = next(c for c in config.contexts if c.name == ctx_name)
        cluster = next(c for c in config.clusters if c.name == ctx.context.cluster).cluster

        if 'certificate-authority' in cluster:
            ca_cert = cluster['certificate-authority']
        else:
            ca_cert = None

        if ctx.context.user:
            user = next(u for u in config.users if u.name == ctx.context.user).user
            return cluster.server, ca_cert, (user['client-certificate'], user['client-key'])
        else:
            return cluster.server, ca_cert, None 

def label_defunct(client: KubernetesAPIClient, namespace: str, job: DotMap):
    job_name = job.metadata.name
    pods = client.get('/api/v1/namespaces/{}/pods', namespace,
                      params={'labelSelector': 'job-name={}'.format(job_name)})

    defunct_ops = [{
        'op': 'add',
        'path': '/metadata/labels/defunct',
        'value': 'true'
    }]

    for pod in pods['items']:
        r = client.json_patch(defunct_ops,
                              '/api/v1/namespaces/{}/pods/{}',
                              namespace, pod.metadata.name,
                              raise_for_status=False)
        if r.status_code != 200:
            # oh well
            logger.error(
                'Failed to label pod as defunct: %s/%s',
                namespace, pod.metadata.name) 

def load_current_kube_credentials():
    with open(os.path.expanduser(KUBE_CONFIG_PATH), 'r') as kcf:
        config = DotMap(yaml.safe_load(kcf))

        ctx_name = config['current-context']
        ctx = next(c for c in config.contexts if c.name == ctx_name)
        cluster = next(c for c in config.clusters if c.name == ctx.context.cluster).cluster

        if 'certificate-authority' in cluster:
            ca_cert = cluster['certificate-authority']
        else:
            ca_cert = None

        if ctx.context.user:
            user = next(u for u in config.users if u.name == ctx.context.user).user
            return cluster.server, ca_cert, (user['client-certificate'], user['client-key'])

        return cluster.server, ca_cert, None 

def test_bayesianOptimization():
    carDomain = Struct({
        'position': Box([-10,10], [-10,10], [0,1]),
        'heading': Box([0, math.pi]),
    })

    space = FeatureSpace({
        'cars': Feature(Array(carDomain, [2]))
    })

    def f(sample):
        sample = sample.cars[0].heading[0]
        return abs(sample - 0.75)

    bo_params = DotMap()
    bo_params.init_num = 2

    sampler = FeatureSampler.bayesianOptimizationSamplerFor(space, BO_params=bo_params)

    sampleWithFeedback(sampler, 3, f) 

def test_feedback_multiple_lengths():
    space = FeatureSpace({
        'a': Feature(Box((0, 1)), lengthDomain=DiscreteBox((1, 2)))
    })

    def f(sample):
        assert 1 <= len(sample.a) <= 2
        return -1 if len(sample.a) == 1 and sample.a[0][0] < 0.5 else 1

    ce_params = DotMap(alpha=0.5, thres=0)
    ce_params.cont.buckets = 2
    ce_params.cont.dist = None
    ce_params.disc.dist = None
    sampler = FeatureSampler.crossEntropySamplerFor(space, ce_params)

    sampleWithFeedback(sampler, 100, f)
    l1sampler = sampler.domainSamplers[sampler.lengthDomain.makePoint(a=(1,))]
    l1dist = l1sampler.cont_sampler.dist[0]
    l2sampler = sampler.domainSamplers[sampler.lengthDomain.makePoint(a=(2,))]
    l2dists = l2sampler.cont_sampler.dist
    assert len(l1dist) == 2
    assert l1dist[0] > 0.9
    assert all(list(l2dist) == [0.5, 0.5] for l2dist in l2dists) 

def read_env_cfg(config_filename = 'configs/main.cfg'):
    # Load from config file
    cfg = DotMap()

    config = cp.ConfigParser()
    config.read(config_filename)

    cfg.run_name = config.get('general_params', 'env_name')
    cfg.floorplan = str(config.get('general_params', 'floorplan'))
    cfg.o_x = float(config.get('general_params', 'o_x').split(',')[0])
    cfg.o_y = float(config.get('general_params', 'o_y').split(',')[0])
    cfg.alpha = float(config.get('general_params', 'alpha').split(',')[0])
    cfg.ceiling_z = float(config.get('general_params', 'ceiling_z').split(',')[0])
    cfg.floor_z = float(config.get('general_params', 'floor_z').split(',')[0])
    cfg.player_start_z = float(config.get('general_params', 'player_start_z').split(',')[0])

    return cfg 

def test_halton():
    carDomain = Struct({
        'position': Box([-10,10], [-10,10], [0,1]),
        'heading': Box([0, math.pi]),
        'model': DiscreteBox([0, 10])
    })

    space = FeatureSpace({
        'weather': Feature(DiscreteBox([0,12])),
        'cars': Feature(Array(carDomain, [2]))
    })

    halton_params = DotMap()
    halton_params.sample_index = -2
    halton_params.bases_skipped = 0

    sampler = FeatureSampler.haltonSamplerFor(space, halton_params)

    for i in range(3):
        print(f'Sample #{i}:')
        print(sampler.nextSample()) 

def __init__(self, params):
        """Initializes an agent.

        Arguments:
            params: (DotMap) A DotMap of agent parameters.
                .env: (OpenAI gym environment) The environment for this agent.
                .noisy_actions: (bool) Indicates whether random Gaussian noise will 
                    be added to the actions of this agent.
                .noise_stddev: (float) The standard deviation to be used for the 
                    action noise if params.noisy_actions is True.
        """
        self.env = params.env
        self.noise_stddev = params.noise_stddev if params.get("noisy_actions", False) else None

        if isinstance(self.env, DotMap):
            raise ValueError("Environment must be provided to the agent at initialization.")
        if (not isinstance(self.noise_stddev, float)) and params.get("noisy_actions", False):
            raise ValueError("Must provide standard deviation for noise for noisy actions.")

        if self.noise_stddev is not None:
            self.dU = self.env.action_space.shape[0] 

def main(env, ctrl_type, ctrl_args, overrides, model_dir, logdir):
    ctrl_args = DotMap(**{key: val for (key, val) in ctrl_args})

    overrides.append(["ctrl_cfg.prop_cfg.model_init_cfg.model_dir", model_dir])
    overrides.append(["ctrl_cfg.prop_cfg.model_init_cfg.load_model", "True"])
    overrides.append(["ctrl_cfg.prop_cfg.model_pretrained", "True"])
    overrides.append(["exp_cfg.exp_cfg.ninit_rollouts", "0"])
    overrides.append(["exp_cfg.exp_cfg.ntrain_iters", "1"])
    overrides.append(["exp_cfg.log_cfg.nrecord", "1"])

    cfg = create_config(env, ctrl_type, ctrl_args, overrides, logdir)
    cfg.pprint()

    if ctrl_type == "MPC":
        cfg.exp_cfg.exp_cfg.policy = MPC(cfg.ctrl_cfg)
    exp = MBExperiment(cfg.exp_cfg)

    os.makedirs(exp.logdir)
    with open(os.path.join(exp.logdir, "config.txt"), "w") as f:
        f.write(pprint.pformat(cfg.toDict()))

    exp.run_experiment() 

def not_a_test_offset(self):
    b = DotMap({ "closed": False,
    "nodes": [
     {"x": 412.0, "y":500.0, "type":"line"},
     {"x": 308.0, "y":665.0, "type":"offcurve"},
     {"x": 163.0, "y":589.0, "type":"offcurve"},
     {"x": 163.0, "y":504.0, "type":"curve"},
     {"x": 163.0, "y":424.0, "type":"offcurve"},
     {"x": 364.0, "y":321.0, "type":"offcurve"},
     {"x": 366.0, "y":216.0, "type":"curve"},
     {"x": 368.0, "y":94.0, "type":"offcurve"},
     {"x": 260.0, "y":54.0, "type":"offcurve"},
     {"x": 124.0, "y":54.0, "type":"curve"}
    ]})
    path = BezierPath()
    path.activeRepresentation = GSPathRepresentation(path,b)
    import matplotlib.pyplot as plt
    fig, ax = plt.subplots()
    path.addExtremes()
    path.plot(ax)
    for n in path.asSegments():
      p = n.tunniPoint
      if p:
        circle = plt.Circle((p.x, p.y), 1, fill=False, color="blue")
        ax.add_artist(circle)
      n.balance()
    path.translate(Point(5,5))
    path.plot(ax, color="red")
    # o1 = path.offset(Point(10,10))
    # o2 = path.offset(Point(-10,-10))
    # o2.reverse()
    # o1.append(o2)
    # o1.plot(ax)
    plt.show() 

def test_BO_oper():
    vals = []
    N = 10
    for k in range(N):
        print("Testing k: ", k)
        def f(sample):
            x = sample.x[0]
            return (6 * x - 2) ** 2 * np.sin(12 * x - 4)

        space = FeatureSpace({'x': Feature(Box([0,1]))})

        bo_params = DotMap()
        bo_params.init_num = 5

        sampler = FeatureSampler.bayesianOptimizationSamplerFor(space, BO_params=bo_params)

        samples = []
        y_samples = []

        feedback = None
        for i in range(20):
            sample = sampler.nextSample(feedback)
            samples.append(sample)
            feedback = f(sample)
            y_samples.append(feedback)

        min_i = np.array(y_samples).argmin()
        vals.append(np.linalg.norm(samples[min_i].x[0]-0.75) < 0.1)

    assert sum(vals)/N >= 0.9 

def test_representations(self):
    b = DotMap({ "closed": True,
    "nodes": [
    {"x":385.0, "y":20.0, "type":"offcurve"},
    { "x":526.0, "y":79.0, "type":"offcurve"},
    { "x":566.0, "y":135.0, "type":"curve"},
    { "x":585.0, "y":162.0, "type":"offcurve"},
    { "x":566.0, "y":260.0, "type":"offcurve"},
    { "x":484.0, "y":281.0, "type":"curve"},
    { "x":484.0, "y":407.0, "type":"offcurve"},
    { "x":381.0, "y":510.0, "type":"offcurve"},
    { "x":255.0, "y":510.0, "type":"curve"},
    { "x":26.0, "y":281.0, "type":"line"},
    { "x":26.0, "y":155.0, "type":"offcurve"},
    { "x":129.0, "y":20.0, "type":"offcurve"},
    { "x":255.0, "y":20.0, "type":"curve"}
    ]})

    path = BezierPath()
    path.activeRepresentation = GSPathRepresentation(path,b)
    nl = path.asNodelist()
    self.assertEqual(len(nl), 13)
    self.assertIsInstance(nl[1], Node)
    self.assertEqual(nl[1].type,"offcurve")
    self.assertAlmostEqual(nl[1].x,526.0)

    segs = path.asSegments()
    self.assertEqual(len(segs), 5)
    self.assertIsInstance(segs[1], CubicBezier)
    self.assertIsInstance(segs[2], Line) 

def from_html(html, url=None, download_date=None):
        """
        Extracts relevant information from an HTML page given as a string. This function does not invoke scrapy but only
        uses the article extractor. If you have the original URL make sure to provide it as this helps NewsPlease
        to extract the publishing date and title.
        :param html:
        :param url:
        :return:
        """
        extractor = article_extractor.Extractor(
            ['newspaper_extractor', 'readability_extractor', 'date_extractor', 'lang_detect_extractor'])

        title_encoded = ''.encode()
        if not url:
            url = ''

        # if an url was given, we can use that as the filename
        filename = urllib.parse.quote_plus(url) + '.json'

        item = NewscrawlerItem()
        item['spider_response'] = DotMap()
        item['spider_response'].body = html
        item['url'] = url
        item['source_domain'] = urllib.parse.urlparse(url).hostname.encode() if url != '' else ''.encode()
        item['html_title'] = title_encoded
        item['rss_title'] = title_encoded
        item['local_path'] = None
        item['filename'] = filename
        item['download_date'] = download_date
        item['modified_date'] = None
        item = extractor.extract(item)

        tmp_article = ExtractedInformationStorage.extract_relevant_info(item)
        final_article = ExtractedInformationStorage.convert_to_class(tmp_article)
        return final_article 

def get_params_from_file(filepath, params_name='params'):
	import importlib
	from dotmap import DotMap
	module = importlib.import_module(filepath)
	params = getattr(module, params_name)
	params = DotMap(params)
	return params 

def main(env, ctrl_type, ctrl_args, overrides, logdir):
    ctrl_args = DotMap(**{key: val for (key, val) in ctrl_args})
    cfg = create_config(env, ctrl_type, ctrl_args, overrides, logdir)
    cfg.pprint()

    if ctrl_type == "MPC":
        cfg.exp_cfg.exp_cfg.policy = MPC(cfg.ctrl_cfg)
    exp = MBExperiment(cfg.exp_cfg)

    os.makedirs(exp.logdir)
    with open(os.path.join(exp.logdir, "config.txt"), "w") as f:
        f.write(pprint.pformat(cfg.toDict()))

    exp.run_experiment() 

def nn_constructor(self, model_init_cfg):
        model = get_required_argument(model_init_cfg, "model_class", "Must provide model class")(DotMap(
            name="model", num_networks=get_required_argument(model_init_cfg, "num_nets", "Must provide ensemble size"),
            sess=self.SESS, load_model=model_init_cfg.get("load_model", False),
            model_dir=model_init_cfg.get("model_dir", None)
        ))
        if not model_init_cfg.get("load_model", False):
            model.add(FC(200, input_dim=self.MODEL_IN, activation="swish", weight_decay=0.00025))
            model.add(FC(200, activation="swish", weight_decay=0.0005))
            model.add(FC(200, activation="swish", weight_decay=0.0005))
            model.add(FC(self.MODEL_OUT, weight_decay=0.00075))
        model.finalize(tf.train.AdamOptimizer, {"learning_rate": 0.001})
        return model 

def nn_constructor(self, model_init_cfg):
        model = get_required_argument(model_init_cfg, "model_class", "Must provide model class")(DotMap(
            name="model", num_networks=get_required_argument(model_init_cfg, "num_nets", "Must provide ensemble size"),
            sess=self.SESS
        ))
        # Construct model below. For example:
        # model.add(FC(*args))
        # ...
        # model.finalize(tf.train.AdamOptimizer, {"learning_rate": 0.001})
        return model 

def nn_constructor(self, model_init_cfg):
        model = get_required_argument(model_init_cfg, "model_class", "Must provide model class")(DotMap(
            name="model", num_networks=get_required_argument(model_init_cfg, "num_nets", "Must provide ensemble size"),
            sess=self.SESS, load_model=model_init_cfg.get("load_model", False),
            model_dir=model_init_cfg.get("model_dir", None)
        ))
        if not model_init_cfg.get("load_model", False):
            model.add(FC(500, input_dim=self.MODEL_IN, activation='swish', weight_decay=0.0001))
            model.add(FC(500, activation='swish', weight_decay=0.00025))
            model.add(FC(500, activation='swish', weight_decay=0.00025))
            model.add(FC(self.MODEL_OUT, weight_decay=0.0005))
        model.finalize(tf.train.AdamOptimizer, {"learning_rate": 0.001})
        return model 

def gp_constructor(self, model_init_cfg):
        model = get_required_argument(model_init_cfg, "model_class", "Must provide model class")(DotMap(
            name="model",
            kernel_class=get_required_argument(model_init_cfg, "kernel_class", "Must provide kernel class"),
            kernel_args=model_init_cfg.get("kernel_args", {}),
            num_inducing_points=get_required_argument(
                model_init_cfg, "num_inducing_points", "Must provide number of inducing points."
            ),
            sess=self.SESS
        ))
        return model 

def nn_constructor(self, model_init_cfg):
        model = get_required_argument(model_init_cfg, "model_class", "Must provide model class")(DotMap(
            name="model", num_networks=get_required_argument(model_init_cfg, "num_nets", "Must provide ensemble size"),
            sess=self.SESS, load_model=model_init_cfg.get("load_model", False),
            model_dir=model_init_cfg.get("model_dir", None)
        ))
        if not model_init_cfg.get("load_model", False):
            model.add(FC(200, input_dim=self.MODEL_IN, activation="swish", weight_decay=0.000025))
            model.add(FC(200, activation="swish", weight_decay=0.00005))
            model.add(FC(200, activation="swish", weight_decay=0.000075))
            model.add(FC(200, activation="swish", weight_decay=0.000075))
            model.add(FC(self.MODEL_OUT, weight_decay=0.0001))
        model.finalize(tf.train.AdamOptimizer, {"learning_rate": 0.001})
        return model 

def gp_constructor(self, model_init_cfg):
        model = get_required_argument(model_init_cfg, "model_class", "Must provide model class")(DotMap(
            name="model",
            kernel_class=get_required_argument(model_init_cfg, "kernel_class", "Must provide kernel class"),
            kernel_args=model_init_cfg.get("kernel_args", {}),
            num_inducing_points=get_required_argument(
                model_init_cfg, "num_inducing_points", "Must provide number of inducing points."
            ),
            sess=self.SESS
        ))
        return model 

def gp_constructor(self, model_init_cfg):
        model = get_required_argument(model_init_cfg, "model_class", "Must provide model class")(DotMap(
            name="model",
            kernel_class=get_required_argument(model_init_cfg, "kernel_class", "Must provide kernel class"),
            kernel_args=model_init_cfg.get("kernel_args", {}),
            num_inducing_points=get_required_argument(
                model_init_cfg, "num_inducing_points", "Must provide number of inducing points."
            ),
            sess=self.SESS
        ))
        return model 

def __init__(self):
        parser = argparse.ArgumentParser(prog='Taranis', description='Taranis server')
        parser.add_argument('--config-file', '-F', dest='config_files', action='append', type=str,
                            help='a list of yaml config files')
        parser.add_argument('--config-path', '-P', dest='config_paths', action='append', type=str,
                            help='a list of directories in which the file paths are keys and file contents are value')
        parser.add_argument('--additional-config', '-C', dest='additional_config', action='append', type=str,
                            help='a list of dotted.key=value configs')
        # args = parser.parse_args()
        args, unknowns = parser.parse_known_args()

        if unknowns:
            parser.print_usage()
            sys.exit(1)

        configurations = list()
        configurations.append(dict(i.split("=") for i in args.additional_config) if args.additional_config else dict())
        configurations.append("env")
        if args.config_paths:
            configurations.extend(args.config_paths)
        if args.config_files:
            configurations.extend(args.config_files)

        conf_set = python_config(*configurations, prefix="TARANIS", remove_level=0)

        tempdict = dict()

        for key, value in conf_set.as_dict().items():
            tree = key.split('.')
            root = tempdict
            for i, b in enumerate(tree):
                if b not in root:
                    if (i + 1) == len(tree):
                        root[b] = value
                    else:
                        root[b] = dict()
                root = root[b]
        self.dict = DotMap(tempdict) 

def analyze(self, analysis_params=None):
        analysis_data = DotMap()
        if analysis_params is None or ('pca' in analysis_params and analysis_params.pca) or 'pca' not in analysis_params:
            if analysis_params is not None and 'pca_params' in analysis_params:
                columns = analysis_params.pca_params.columns \
                    if 'columns' in analysis_params.pca_params else None
                n_components = analysis_params.pca_params.n_components \
                    if 'n_components' in analysis_params.pca_params else 1
            else:
                columns, n_components = None, 1
            analysis_data.pca = self.pca_analysis(column_names=columns, n_components=n_components)

        if analysis_params is None or ('k_closest' in analysis_params and analysis_params.k_closest) or 'k_closest' not in analysis_params:
            if analysis_params is not None and 'k_closest_params' in analysis_params:
                columns = analysis_params.k_closest_params.columns \
                    if 'columns' in analysis_params.k_closest_params else None
                k = analysis_params.k_closest_params.k \
                    if 'k' in analysis_params.k_closest_params else None
            else:
                columns, k = None, None
            analysis_data.k_closest = self.k_closest_samples(column_names=columns, k=k)


        if analysis_params is None or ('random' in analysis_params and analysis_params.random) or 'random' not in analysis_params:
            if analysis_params is not None and 'random_params' in analysis_params:
                count = analysis_params.random_params.count \
                    if 'count' in analysis_params.random_params else 5
            else:
                count = 5
            analysis_data.random = self.get_random_samples(count=count)

        if analysis_params is None or ('k_clusters' in analysis_params and analysis_params.k_clusters) or 'k_clusters' not in analysis_params:
            if analysis_params is not None and 'k_clusters_params' in analysis_params:
                columns = analysis_params.k_clusters_params.columns \
                    if 'columns' in analysis_params.k_clusters_params else None
                k = analysis_params.k_clusters_params.k \
                    if 'k' in analysis_params.k_clusters_params else None
            else:
                columns, k = None, None
            analysis_data.k_clusters = self.k_clusters(column_names=columns, k=k)
        return analysis_data 

def read_config(path):
    if path:
        with open(path) as json_data_file:
            conf_args = json.load(json_data_file)
    else:
        raise ValueError("No config provided for classifier")

    # flatten last part of config, take either value or default as value
    for gk, gv in conf_args.items():
        for k, v in gv.items():
            conf_args[gk][k] = v["value"] if (v["value"] is not None) else v["default"]

    # DotMap for making nested dictionary accessible through dot notation
    args = DotMap(conf_args, _dynamic=False)

    return args 

def is_condition_complete(condition: DotMap) -> bool:
    return condition.type == 'Complete' and str(condition.status) == 'True' 

def get_config_from_json(json_file):
    """
    Get the config from a json file
    :param json_file:
    :return: config(namespace) or config(dictionary)
    """
    # parse the configurations from the config json file provided
    with open(json_file, 'r') as config_file:
        config_dict = json.load(config_file)

    # convert the dictionary to a namespace using bunch lib
    config = DotMap(config_dict)

    return config, config_dict 

def __init__(self, baselines_params=None):
        if baselines_params is None:
            baselines_params = DotMap()
            baselines_params.alg = 'ppo2'
            baselines_params.env_id = 'MountainCar-v0'
            baseline_params.num_timesteps = 1e3
        else:
            if 'env_id' not in baseline_params or baseline_params.env_id !='MountainCar-v0':
                baseline_params.env_id = 'MountainCar-v0'
            if 'alg' not in baseline_params:
                baseline_params.alg = 'ppo2'
        super().__init__(baselines_params=baselines_params)
        if sample_type >= 1:
            self.run_task = self.run_task_retrain
        self.algs = ['ppo2', 'deepq', 'acer', 'a2c', 'trpo_mpi', 'acktr'] 

def __init__(self, baselines_params=None):
        if baselines_params is None:
            baselines_params = DotMap()
            baselines_params.alg = 'ppo2'
            baselines_params.env_id = 'CartPole-v1'
            baseline_params.num_timesteps = 1e5
        else:
            if 'env_id' not in baseline_params or baseline_params.env_id !='CartPole-v1':
                baseline_params.env_id = 'CartPole-v1'
            if 'alg' not in baseline_params:
                baseline_params.alg = 'ppo2'
        super().__init__(baselines_params=baselines_params)
        if sample_type >= 1:
            self.run_task = self.run_task_retrain
        self.algs = ['ppo2', 'deepq', 'acer', 'a2c', 'trpo_mpi', 'acktr']