Python attrdict.AttrDict() Examples

def main():
    config_path = './config.json'
    with open(config_path) as f:
        args = json.load(f)
        args = AttrDict(args)
    device = torch.device(args.device)
    args.model = nn.chimera(**(args['model_options']))
    args.model.to(device)
    args.train_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'tr', device)
    args.valid_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'cv', device)
    args.test_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'tt', device)
    args.optimizer = utils.build_optimizer(args.model.parameters(), args.optimizer_options)
    args.loss_fn = loss.loss_chimera_msa
    trainer = utils.trainer(args)
    trainer.run()
    tester = tester_chimera(args)
    tester.eval() 

def __init__(self, train_mode, loader_params, dataset_params, augmentation_params):
        super().__init__()
        self.train_mode = train_mode
        self.loader_params = AttrDict(loader_params)
        self.dataset_params = AttrDict(dataset_params)
        self.augmentation_params = AttrDict(augmentation_params)

        self.mask_transform = None
        self.image_transform = None

        self.image_augment_train = None
        self.image_augment_inference = None
        self.image_augment_with_target_train = None
        self.image_augment_with_target_inference = None

        self.dataset = None 

def transform(self, X):
        try:
            res = []
            for idx, row in tqdm(X.iterrows(), total=len(X)):
                res.append(self.tokenizer.tokenize(**row)[1:])

            res = pd.DataFrame(res, columns=['tokens', 'pronoun_offset_token',
                                                    'a_offset_token', 'b_offset_token', 'a_span',
                                                    'b_span', 'pronoun_token', 'a_tokens', 'b_tokens'])

            cols = set(X.columns).difference(res.columns)
            X = pd.concat([X[cols], res], axis=1)
            return AttrDict({'X': X})
        except Exception as e:
            print(row.text)
            raise e 

def example_to_debug(self, X, idx):
        ex = AttrDict(X['X'].to_dict(orient='records')[idx])
        
        text = ex.text
        text = '{}<A>{}'.format(text[:ex.a_offset], text[ex.a_offset:])
        text = '{}<B>{}'.format(text[:ex.b_offset+3], text[ex.b_offset+3:])
        
        offset = ex.pronoun_offset
        if ex.pronoun_offset > ex.a_offset:
            offset += 3
        if ex.pronoun_offset > ex.b_offset:
            offset += 3
            
        text = '{}<P>{}'.format(text[:offset], text[offset:])

        ex.a_offset = text.index('<A>')
        ex.b_offset = text.index('<B>')
        ex.pronoun_offset = text.index('<P>')

        ex.text = text
        
        return ex 

def __init__(self):

        self.font_width = 7
        self.lane = AttrDict({
            "xs": 20,    # tmpgraphlane0.width
            "ys": 20,    # tmpgraphlane0.height
            "xg": 120,   # tmpgraphlane0.x
            "yg": 0,     # head gap
            "yh0": 0,     # head gap title
            "yh1": 0,     # head gap
            "yf0": 0,     # foot gap
            "yf1": 0,     # foot gap
            "y0": 5,     # tmpgraphlane0.y
            "yo": 30,    # tmpgraphlane1.y - y0
            "tgo": -10,   # tmptextlane0.x - xg
            "ym": 15,    # tmptextlane0.y - y0
            "xlabel": 6,     # tmptextlabel.x - xg
            "xmax": 1,
            "scale": 1,
            "head": {},
            "foot": {}
        }) 

def __init__(self, train_mode, loader_params, dataset_params, augmentation_params):
        super().__init__()
        self.train_mode = train_mode
        self.loader_params = AttrDict(loader_params)
        self.dataset_params = AttrDict(dataset_params)
        self.augmentation_params = AttrDict(augmentation_params)

        self.mask_transform = None
        self.image_transform = None

        self.image_augment_train = None
        self.image_augment_inference = None
        self.image_augment_with_target_train = None
        self.image_augment_with_target_inference = None

        self.dataset = None 

def exec(self, cmd: str, quiet=False, supress_error=False) -> AttrDict:
        ret = AttrDict()

        if not quiet:
            self.log.debug(cmd)

        p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=True)

        # output processing
        out = p.stdout.read().decode().strip()
        ret.out = out
        err = p.stderr.read().decode().strip()
        ret.err = err

        output = '{} <output> {}'.format(cmd, out if out else 'Nothing')

        if err and not supress_error:
            output += ' <error> ' + err

        if not quiet:
            self.log.debug(output)

        return ret 

def load(config, **unused_kwargs):

    del unused_kwargs

    if not os.path.exists(config.data_folder):
        os.makedirs(config.data_folder)

    dataset = input_data.read_data_sets(config.data_folder)

    train_data = {'imgs': dataset.train.images, 'labels': dataset.train.labels}
    valid_data = {'imgs': dataset.validation.images, 'labels': dataset.validation.labels}

    # This function turns a dictionary of numpy.ndarrays into tensors.
    train_tensors = tensors_from_data(train_data, config.batch_size, shuffle=True)
    valid_tensors = tensors_from_data(valid_data, config.batch_size, shuffle=False)

    data_dict = AttrDict(
        train_img=train_tensors['imgs'],
        valid_img=valid_tensors['imgs'],
        train_label=train_tensors['labels'],
        valid_label=valid_tensors['labels'],
    )

    return data_dict 

def load(config, **unused_kwargs):

    del unused_kwargs

    if not os.path.exists(config.data_folder):
        os.makedirs(config.data_folder)

    dataset = input_data.read_data_sets(config.data_folder)

    train_data = {'imgs': dataset.train.images, 'labels': dataset.train.labels}
    valid_data = {'imgs': dataset.validation.images, 'labels': dataset.validation.labels}

    train_tensors = tensors_from_data(train_data, config.batch_size, shuffle=True)
    valid_tensors = tensors_from_data(valid_data, config.batch_size, shuffle=False)

    data_dict = AttrDict(
        train_img=train_tensors['imgs'],
        valid_img=valid_tensors['imgs'],
        train_label=train_tensors['labels'],
        valid_label=valid_tensors['labels'],
    )

    return data_dict 

def __init__(self, strings=6, frets=(0, 5), inlays=None, style=None):
        self.frets = list(range(max(frets[0] - 1, 0), frets[1] + 1))
        self.strings = [attrdict.AttrDict({
            'color': None,
            'label': None,
            'font_color': None,
        }) for x in range(strings)]

        self.markers = []

        self.inlays = inlays if inlays is not None else self.inlays

        self.layout = attrdict.AttrDict()

        self.style = attrdict.AttrDict(
            dict_merge(
                copy.deepcopy(self.default_style),
                style or {}
            )
        ) 

def __init__(self, train_mode, loader_params, dataset_params):
        super().__init__()
        self.train_mode = train_mode
        self.loader_params = AttrDict(loader_params)
        self.dataset_params = AttrDict(dataset_params)

        sampler_name = self.dataset_params.sampler_name
        if sampler_name == 'fixed':
            self.sampler = FixedSizeSampler
        elif sampler_name == 'aspect ratio':
            self.sampler = AspectRatioSampler
        else:
            msg = "expected sampler name from (fixed, aspect ratio), got {} instead".format(sampler_name)
            raise Exception(msg)

        self.target_encoder = DataEncoder(**self.dataset_params.data_encoder)
        self.dataset = ImageDetectionDataset

        self.image_transform = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize(mean=MEAN, std=STD),
        ])
        self.image_augment = aug_seq 

def get_generator(checkpoint):
    args = AttrDict(checkpoint['args'])
    generator = TrajectoryGenerator(
        obs_len=args.obs_len,
        pred_len=args.pred_len,
        embedding_dim=args.embedding_dim,
        encoder_h_dim=args.encoder_h_dim_g,
        decoder_h_dim=args.decoder_h_dim_g,
        mlp_dim=args.mlp_dim,
        num_layers=args.num_layers,
        noise_dim=args.noise_dim,
        noise_type=args.noise_type,
        noise_mix_type=args.noise_mix_type,
        pooling_type=args.pooling_type,
        pool_every_timestep=args.pool_every_timestep,
        dropout=args.dropout,
        bottleneck_dim=args.bottleneck_dim,
        neighborhood_size=args.neighborhood_size,
        grid_size=args.grid_size,
        batch_norm=args.batch_norm)
    generator.load_state_dict(checkpoint['g_state'])
    generator.cuda()
    generator.train()
    return generator 

def main(args):
    if os.path.isdir(args.model_path):
        filenames = os.listdir(args.model_path)
        filenames.sort()
        paths = [
            os.path.join(args.model_path, file_) for file_ in filenames
        ]
    else:
        paths = [args.model_path]

    for path in paths:
        checkpoint = torch.load(path)
        generator = get_generator(checkpoint)
        _args = AttrDict(checkpoint['args'])
        path = get_dset_path(_args.dataset_name, args.dset_type)
        _, loader = data_loader(_args, path)
        ade, fde = evaluate(_args, loader, generator, args.num_samples)
        print('Dataset: {}, Pred Len: {}, ADE: {:.2f}, FDE: {:.2f}'.format(
            _args.dataset_name, _args.pred_len, ade, fde)) 

def main():
    parser = argparse.ArgumentParser(description='Parse the config path')
    parser.add_argument("-c", "--config", dest="path",
                        help='The path to the config file. e.g. python run.py --config dc_config.json')

    config = parser.parse_args()
    with open(config.path) as f:
        args = json.load(f)
        args = AttrDict(args)
    device = torch.device(args.device)
    args.model = onssen.nn.chimera(args.model_options)
    args.model.to(device)
    args.train_loader = data.edinburgh_tts_dataloader(args.model_name, args.feature_options, 'train', args.cuda_option, self.device)
    args.valid_loader = data.edinburgh_tts_dataloader(args.model_name, args.feature_options, 'validation', args.cuda_option, self.device)
    args.optimizer = utils.build_optimizer(args.model.parameters(), args.optimizer_options)
    args.loss_fn = loss.loss_chimera_psa
    trainer = onssen.utils.trainer(args)
    trainer.run()

    tester = onssen.utils.tester(args)
    tester.eval() 

def main():
    config_path = './config.json'
    with open(config_path) as f:
        args = json.load(f)
        args = AttrDict(args)
    device = torch.device(args.device)
    args.device = device
    args.model = nn.ConvTasNet(**args["model_options"])
    args.model.to(device)
    args.train_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'tr', device)
    args.valid_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'cv', device)
    args.test_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'tt', device)
    args.optimizer = utils.build_optimizer(args.model.parameters(), args.optimizer_options)
    args.loss_fn = loss.si_snr_loss
    trainer = utils.trainer(args)
    trainer.run()
    tester = tester_tasnet(args)
    tester.eval() 

def main():
    parser = argparse.ArgumentParser(description='Parse the config path')
    parser.add_argument("-c", "--config", dest="path",
                        help='The path to the config file. e.g. python run.py --config onfig.json')

    config = parser.parse_args()
    with open(config.path) as f:
        args = json.load(f)
        args = AttrDict(args)
    device = torch.device(args.device)
    args.model = nn.deep_clustering(**(args['model_options']))
    args.model.to(device)
    args.train_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'tr', device)
    args.valid_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'cv', device)
    args.test_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'tt', device)
    args.optimizer = utils.build_optimizer(args.model.parameters(), args.optimizer_options)
    args.loss_fn = loss.loss_dc
    trainer = utils.trainer(args)
    trainer.run()

    tester = tester_dc(args)
    tester.eval() 

def __init__(self, positions=None, fingers=None, style=None):
        if positions is None:
            positions = []
        elif '-' in positions:
            positions = positions.split('-')
        else:
            positions = list(positions)
        self.positions = list(map(lambda p: int(p) if p.isdigit() else None, positions))

        self.fingers = list(fingers) if fingers else []

        self.style = attrdict.AttrDict(
            dict_merge(
                copy.deepcopy(self.default_style),
                style or {}
            )
        ) 

def main():
    config_path = './config.json'
    with open(config_path) as f:
        args = json.load(f)
        args = AttrDict(args)
    device = torch.device(args.device)
    args.model = nn.chimera(**(args['model_options']))
    args.model.to(device)
    args.train_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'tr', device)
    args.valid_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'cv', device)
    args.test_loader = data.wsj0_2mix_dataloader(args.model_name, args.feature_options, 'tt', device)
    args.optimizer = utils.build_optimizer(args.model.parameters(), args.optimizer_options)
    args.loss_fn = loss.loss_chimera_psa
    trainer = utils.trainer(args)
    trainer.run()
    tester = tester_chimera(args)
    tester.eval() 

def __init__(self, train_mode, loader_params, dataset_params, augmentation_params):
        super().__init__()
        self.train_mode = train_mode
        self.loader_params = AttrDict(loader_params)
        self.dataset_params = AttrDict(dataset_params)
        self.augmentation_params = AttrDict(augmentation_params)

        self.mask_transform = None
        self.image_transform = None

        self.image_augment_train = None
        self.image_augment_inference = None
        self.image_augment_with_target_train = None
        self.image_augment_with_target_inference = None

        self.dataset = None 

def model_config(self):
        return AttrDict({param: value for param, value in self.params.items()
                         if param not in self.training_params}) 

def read_yaml(filepath):
    with open(filepath) as f:
        config = yaml.load(f)
    return AttrDict(config) 

def get_random_config(tuning_config):
        config_run = {}
        for tunable_name in tuning_config.keys():
            param_choice = {}
            for param, value in tuning_config[tunable_name].items():
                param_range, sampling_mode = value
                param_choice[param] = RandomSearchTuner.random_sample_from_param_space(param_range, sampling_mode)
            config_run[tunable_name] = param_choice
        return AttrDict(config_run) 

def main():
    args = parse_args()
    work_dir = Path(args.work_dir) / args.id
    engine = nemo.core.NeuralModuleFactory(
        local_rank=args.local_rank,
        optimization_level=args.amp_opt_level,
        cudnn_benchmark=args.cudnn_benchmark,
        log_dir=work_dir / 'log',
        checkpoint_dir=work_dir / 'checkpoints',
        tensorboard_dir=work_dir / 'tensorboard',
        files_to_copy=[args.model_config],
    )

    yaml_loader = yaml.YAML(typ="safe")
    with open(args.model_config) as f:
        config = attrdict.AttrDict(yaml_loader.load(f))
    logging.info(f'Config: {config}')
    graph = FastSpeechGraph(args, config, num_workers=max(int(os.cpu_count() / engine.world_size), 1))

    steps_per_epoch = math.ceil(len(graph.data_layer) / (args.batch_size * engine.world_size))
    total_steps = args.max_steps if args.max_steps is not None else args.num_epochs * steps_per_epoch
    loss, callbacks = graph.build()
    engine.train(
        tensors_to_optimize=[loss],
        optimizer=args.optimizer,
        optimization_params=dict(
            num_epochs=args.num_epochs,
            max_steps=total_steps,
            lr=args.lr,
            weight_decay=args.weight_decay,
            grad_norm_clip=args.grad_norm_clip,
        ),
        callbacks=callbacks,
        lr_policy=lr_policies.CosineAnnealing(total_steps, min_lr=args.min_lr, warmup_steps=4000),
    ) 

def __init__(self, model_params, training_params):
        self.model_params = model_params
        self.training_params = AttrDict(training_params)
        self.evaluation_function = None 

def __init__(self, model_config, training_config):
        self.model_config = AttrDict(model_config)
        self.training_config = AttrDict(training_config)
        self.evaluation_function = None 

def read_config(config_path):
    with open(config_path) as f:
        config = yaml.load(f)
    return AttrDict(config) 

def __init__(self, loader_params, dataset_params):
        super().__init__()
        self.loader_params = AttrDict(loader_params)
        self.dataset_params = AttrDict(dataset_params)

        self.image_transform = None
        self.mask_transform = None

        self.image_augment_with_target_train = None
        self.image_augment_with_target_inference = None
        self.image_augment_train = None
        self.image_augment_inference = None

        self.dataset = None 

def __init__(self, **kwargs):
        self.tta_transformations = AttrDict(kwargs) 

def test_set_experiment_with_zero_id(reset_mock):
    reset_mock(MlflowClient, "get_experiment_by_name",
               mock.Mock(return_value=attrdict.AttrDict(
                   experiment_id=0,
                   lifecycle_stage=LifecycleStage.ACTIVE)))
    reset_mock(MlflowClient, "create_experiment", mock.Mock())

    mlflow.set_experiment("my_exp")

    MlflowClient.get_experiment_by_name.assert_called_once()
    MlflowClient.create_experiment.assert_not_called() 

def read_yaml(filepath):
    with open(filepath) as f:
        config = yaml.load(f)
    return AttrDict(config)