Python config.load_config() Examples

def config_initialization():
    # image shape and feature layers shape inference
    image_shape = (FLAGS.eval_image_height, FLAGS.eval_image_width)
    
    if not FLAGS.dataset_dir:
        raise ValueError('You must supply the dataset directory with --dataset_dir')
    
    tf.logging.set_verbosity(tf.logging.DEBUG)
    config.load_config(FLAGS.checkpoint_path)
    config.init_config(image_shape, 
                       batch_size = 1, 
                       pixel_conf_threshold = 0.8,
                       link_conf_threshold = 0.8,
                       num_gpus = 1, 
                   )
    
    util.proc.set_proc_name('test_pixel_link_on'+ '_' + FLAGS.dataset_name) 

def config_initialization():
    # image shape and feature layers shape inference
    image_shape = (FLAGS.eval_image_height, FLAGS.eval_image_width)
    
    if not FLAGS.dataset_dir:
        raise ValueError('You must supply the dataset directory with --dataset_dir')
    
    tf.logging.set_verbosity(tf.logging.DEBUG)
    config.load_config(FLAGS.checkpoint_path)
    config.init_config(image_shape, 
                       batch_size = 1, 
                       pixel_conf_threshold = 0.8,
                       link_conf_threshold = 0.8,
                       num_gpus = 1, 
                   )
    
    util.proc.set_proc_name('test_pixel_link_on'+ '_' + FLAGS.dataset_name) 

def add():
    if not check_num_args(3): return

    # load config and others
    config = Config.load_config()
    username = sys.argv[2].lower()
    idx = Config.find_in_config(username, config)

    # if already added
    if idx:
        print("{} has already been added".format(username))
        return

    # add and save
    config["streamers"].append(username)
    if Config.save_config(config):
        print("{} has been added".format(username)) 

def remove():
    if not check_num_args(3): return
    
    # load config and others
    config = Config.load_config()
    username = sys.argv[2].lower()
    idx = Config.find_in_config(username, config)

    # if not in list
    if idx == None:
        print("{} hasn't been added".format(username))
        return

    # delete and save
    del config["streamers"][idx]
    if Config.save_config(config):
        print("{} has been deleted".format(username)) 

def import_streamers():
    if not check_num_args(3): return

    # load config
    config = Config.load_config()
    
    # open and loop file
    with open(sys.argv[2], "r") as f:
        for line in f:
            username = line.rstrip()

            # if already in, print, else append
            if username in config["streamers"]:
                print("{} has already been added".format(username))
            else:
                config["streamers"].append(username)
    
    # save config
    if Config.save_config(config):
        print("Streamers imported, Config saved") 

def export_streamers():
    if len(sys.argv) not in (2, 3): return usage()

    # load config
    config = Config.load_config()
    export_location = config["default_export_location"]

    # check export loc
    if len(sys.argv) == 3:
        export_location = sys.argv[2]

    # write file
    with open(export_location, "w") as f:
        for streamer in config["streamers"]:
            f.write(streamer + "\n")
    
    print("Written streamers to file") 

def create_container(c):
    """Creates container based on the parameters found in the .env file
    """
    logger = logging.getLogger(__name__)
    env_values = load_config()
    container_name = env_values.get("CONTAINER_NAME")
    account_name = env_values.get("ACCOUNT_NAME")
    account_key = env_values.get("ACCOUNT_KEY")
    if _container_exists(c, container_name, account_name, account_key):
        logger.info(f"Container already exists")
        return None

    cmd = (
        f"az storage container create"
        f" --account-name {account_name}"
        f" --account-key {account_key}"
        f" --name {container_name}"
    )
    c.run(cmd) 

def __init__(self, image_dir, config_path, weights_path, threshold=0.5, class_names=None):
        super(GuiInferenceViewer, self).__init__('Inference Viewer')
        assert os.path.isdir(image_dir), 'No such directory: {}'.format(image_dir)

        self.threshold = threshold
        self.class_names = class_names
        # Get the list of image files
        self.image_files = sorted([os.path.join(image_dir, x)
                            for x in os.listdir(image_dir)
                            if x.lower().endswith('.jpg') or x.lower().endswith('.png') or x.lower().endswith('.bmp')
                            ])
        self.num_images = len(self.image_files)

        self.model = get_model_wrapper(load_config(config_path))
        if weights_path:
            self.model.load_weights(weights_path)
        else:
            print('No weights path provided. Will use randomly initialized weights')

        self.create_slider()
        self.create_textbox()

        self.display() 

def authenticate_user(user_name):
    user_items = None
    if os.path.isfile(GLOBAL_CONFIG_FILEPATH):
        global_config_object = config.load_config(GLOBAL_CONFIG_FILEPATH)
        if user_name in global_config_object.sections():
            user_items = config.load_user(global_config_object, user_name)
            return user_items
    user_items = prompt_user_for_user_items(user_name)
    if not user_items:
        return None
    try:
        config.create_config(GLOBAL_CONFIG_FILEPATH)
    except Exception as e:
        print("Failed to create authentication config file due to {}".format(e))
        sys.exit(1)
    config.update_config(
        GLOBAL_CONFIG_FILEPATH, user_name, user_items)
    return user_items 

def upload_validation_data(c):
    """Upload validation data to container specified in .env file
    """
    env_values = load_config()
    container_name = env_values.get("CONTAINER_NAME")
    account_name = env_values.get("ACCOUNT_NAME")
    account_key = env_values.get("ACCOUNT_KEY")
    upload_data_from_to(
        c, "validation", "/data/validation", container_name, account_name, account_key
    ) 

def get_master_key():
    master_key = ""
    if os.path.isfile(GLOBAL_CONFIG_FILEPATH):
        global_config_object = config.load_config(GLOBAL_CONFIG_FILEPATH)
        if "MAPAdmin" in global_config_object.sections():
            admin_items = config.load_user(global_config_object, "MAPAdmin")
            if "MAPILLARY_SECRET_HASH" in admin_items:
                master_key = admin_items["MAPILLARY_SECRET_HASH"]
            else:
                create_config = raw_input(
                    "Master upload key does not exist in your global Mapillary config file, set it now?")
                if create_config in ["y", "Y", "yes", "Yes"]:
                    master_key = set_master_key()
        else:
            create_config = raw_input(
                "MAPAdmin section not in your global Mapillary config file, set it now?")
            if create_config in ["y", "Y", "yes", "Yes"]:
                master_key = set_master_key()
    else:
        create_config = raw_input(
            "Master upload key needs to be saved in the global Mapillary config file, which does not exist, create one now?")
        if create_config in ["y", "Y", "yes", "Yes"]:
            config.create_config(GLOBAL_CONFIG_FILEPATH)
            master_key = set_master_key()

    return master_key 

def download_training(c):
    """Download training data from blob container specified in .env file
    """
    env_values = load_config()
    container_name = env_values.get("CONTAINER_NAME")
    account_name = env_values.get("ACCOUNT_NAME")
    account_key = env_values.get("ACCOUNT_KEY")
    download_data_from_to(
        c, "train", "/data/train", container_name, account_name, account_key
    ) 

def download_validation(c):
    """Download validation data from blob container specified in .env file
    """
    env_values = load_config()
    container_name = env_values.get("CONTAINER_NAME")
    account_name = env_values.get("ACCOUNT_NAME")
    account_key = env_values.get("ACCOUNT_KEY")
    download_data_from_to(
        c, "validation", "/data/validation", container_name, account_name, account_key
    ) 

def upload_validation_data(c):
    """Upload tfrecords validation data to container specified in .env file
    """
    env_values = load_config()
    container_name = env_values.get("CONTAINER_NAME")
    account_name = env_values.get("ACCOUNT_NAME")
    account_key = env_values.get("ACCOUNT_KEY")
    upload_data_from_to(
        c,
        "tfrecords/validation",
        "/data/tfrecords/validation",
        container_name,
        account_name,
        account_key,
    ) 

def download_training(c):
    """Download tfrecords training data from blob container specified in .env file
    """
    env_values = load_config()
    container_name = env_values.get("CONTAINER_NAME")
    account_name = env_values.get("ACCOUNT_NAME")
    account_key = env_values.get("ACCOUNT_KEY")
    download_data_from_to(
        c,
        "tfrecords/train",
        "/data/tfrecords/train",
        container_name,
        account_name,
        account_key,
    ) 

def download_validation(c):
    """Download tfrecords validation data from blob container specified in .env file
    """
    env_values = load_config()
    container_name = env_values.get("CONTAINER_NAME")
    account_name = env_values.get("ACCOUNT_NAME")
    account_key = env_values.get("ACCOUNT_KEY")
    download_data_from_to(
        c,
        "tfrecords/validation",
        "/data/tfrecords/validation",
        container_name,
        account_name,
        account_key,
    ) 

def config_initialization():
    # image shape and feature layers shape inference
    image_shape = (FLAGS.train_image_height, FLAGS.train_image_width)
    
    if not FLAGS.dataset_dir:
        raise ValueError('You must supply the dataset directory with --dataset_dir')
    
    tf.logging.set_verbosity(tf.logging.DEBUG)
    util.init_logger(
        log_file = 'log_train_pixel_link_%d_%d.log'%image_shape, 
                    log_path = FLAGS.train_dir, stdout = False, mode = 'a')
    
    
    config.load_config(FLAGS.train_dir)
            
    config.init_config(image_shape, 
                       batch_size = FLAGS.batch_size, 
                       weight_decay = FLAGS.weight_decay, 
                       num_gpus = FLAGS.num_gpus
                   )

    batch_size = config.batch_size
    batch_size_per_gpu = config.batch_size_per_gpu
        
    tf.summary.scalar('batch_size', batch_size)
    tf.summary.scalar('batch_size_per_gpu', batch_size_per_gpu)

    util.proc.set_proc_name('train_pixel_link_on'+ '_' + FLAGS.dataset_name)
    
    dataset = dataset_factory.get_dataset(FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir)
    config.print_config(FLAGS, dataset)
    return dataset 

def upload_training_data(c):
    """Upload training data to container specified in .env file
    """
    env_values = load_config()
    container_name = env_values.get("CONTAINER_NAME")
    account_name = env_values.get("ACCOUNT_NAME")
    account_key = env_values.get("ACCOUNT_KEY")
    upload_data_from_to(
        c, "train", "/data/train", container_name, account_name, account_key
    ) 

def config_initialization():
    # image shape and feature layers shape inference
    image_shape = (FLAGS.train_image_height, FLAGS.train_image_width)
    
    if not FLAGS.dataset_dir:
        raise ValueError('You must supply the dataset directory with --dataset_dir')
    
    tf.logging.set_verbosity(tf.logging.DEBUG)
    util.init_logger(
        log_file = 'log_train_pixel_link_%d_%d.log'%image_shape, 
                    log_path = FLAGS.train_dir, stdout = False, mode = 'a')
    
    
    config.load_config(FLAGS.train_dir)
            
    config.init_config(image_shape, 
                       batch_size = FLAGS.batch_size, 
                       weight_decay = FLAGS.weight_decay, 
                       num_gpus = FLAGS.num_gpus
                   )

    batch_size = config.batch_size
    batch_size_per_gpu = config.batch_size_per_gpu
        
    tf.summary.scalar('batch_size', batch_size)
    tf.summary.scalar('batch_size_per_gpu', batch_size_per_gpu)

    util.proc.set_proc_name('train_pixel_link_on'+ '_' + FLAGS.dataset_name)
    
    dataset = dataset_factory.get_dataset(FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir)
    config.print_config(FLAGS, dataset)
    return dataset 

def set_master_key():
    config_object = config.load_config(GLOBAL_CONFIG_FILEPATH)
    section = "MAPAdmin"
    if section not in config_object.sections():
        config_object.add_section(section)
    master_key = raw_input("Enter the master key : ")
    if master_key != "":
        config_object = config.set_user_items(
            config_object, section, {"MAPILLARY_SECRET_HASH": master_key})
        config.save_config(config_object, GLOBAL_CONFIG_FILEPATH)
    return master_key 

def app(ctx, alt_config, config_values, data_dir, log_config, bootstrap_node, log_json, mining_pct):

    # configure logging
    log_config = log_config or ':info'
    slogging.configure(log_config, log_json=log_json)

    # data dir default or from cli option
    data_dir = data_dir or konfig.default_data_dir
    konfig.setup_data_dir(data_dir)  # if not available, sets up data_dir and required config
    log.info('using data in', path=data_dir)

    # prepare configuration
    # config files only contain required config (privkeys) and config different from the default
    if alt_config:  # specified config file
        config = konfig.load_config(alt_config)
    else:  # load config from default or set data_dir
        config = konfig.load_config(data_dir)

    config['data_dir'] = data_dir

    # add default config
    konfig.update_config_with_defaults(config, konfig.get_default_config([EthApp] + services))

    # override values with values from cmd line
    for config_value in config_values:
        try:
            konfig.set_config_param(config, config_value)
            # check if this is part of the default config
        except ValueError:
            raise BadParameter('Config parameter must be of the form "a.b.c=d" where "a.b.c" '
                               'specifies the parameter to set and d is a valid yaml value '
                               '(example: "-c jsonrpc.port=5000")')
    if bootstrap_node:
        config['discovery']['bootstrap_nodes'] = [bytes(bootstrap_node)]

    if mining_pct > 0:
        config['pow']['activated'] = True
        config['pow']['cpu_pct'] = int(min(100, mining_pct))

    ctx.obj = {'config': config} 

def main(args):
    config = load_config(args)
    global_train_config = config["training_params"]
    models, model_names = config_modelloader_and_convert2mlp(config) 

def list_streamers():
    if not check_num_args(2): return

    # load config, print streamers
    config = Config.load_config()
    print('Streamers in recording list:\n')
    for streamer in config['streamers']:
        print('- ' + streamer) 

def pytest_configure(config):
    """ Load Test Suite Configuration. """
    dirname = os.getcwd()
    config_path = config.getoption('suite_config')
    config_path = 'config.toml' if not config_path else config_path
    config_path = os.path.join(dirname, config_path)
    print(
        '\nAttempting to load configuration from file: %s\n' %
        config_path
    )

    try:
        config.suite_config = load_config(config_path)
    except FileNotFoundError:
        config.suite_config = default()
    config.suite_config['save_path'] = config.getoption('save_path')

    # Override default terminal reporter for better test output when not capturing
    if config.getoption('capture') == 'no':
        reporter = config.pluginmanager.get_plugin('terminalreporter')
        agent_reporter = AgentTerminalReporter(config, sys.stdout)
        config.pluginmanager.unregister(reporter)
        config.pluginmanager.register(agent_reporter, 'terminalreporter')

    # Compile select regex and test regex if given
    select_regex = config.getoption('select')
    config.select_regex = re.compile(select_regex) if select_regex else None
    config.tests_regex = list(map(
        re.compile, config.suite_config['tests']
    )) 

def main(args):
    config = load_config(args)
    global_eval_config = config["eval_params"]
    models, model_names = config_modelloader(config, load_pretrain = True)

    robust_errs = []
    errs = []

    for model, model_id, model_config in zip(models, model_names, config["models"]):
        # make a copy of global training config, and update per-model config
        eval_config = copy.deepcopy(global_eval_config)
        if "eval_params" in model_config:
            eval_config.update(model_config["eval_params"])

        model = BoundSequential.convert(model, eval_config["method_params"]["bound_opts"]) 
        model = model.cuda()
        # read training parameters from config file
        method = eval_config["method"]
        verbose = eval_config["verbose"]
        eps = eval_config["epsilon"]
        # parameters specific to a training method
        method_param = eval_config["method_params"]
        norm = float(eval_config["norm"])
        train_data, test_data = config_dataloader(config, **eval_config["loader_params"])

        model_name = get_path(config, model_id, "model", load = False)
        print(model_name)
        model_log = get_path(config, model_id, "eval_log")
        logger = Logger(open(model_log, "w"))
        logger.log("evaluation configurations:", eval_config)
            
        logger.log("Evaluating...")
        with torch.no_grad():
            # evaluate
            robust_err, err = Train(model, 0, test_data, EpsilonScheduler("linear", 0, 0, eps, eps, 1), eps, norm, logger, verbose, False, None, method, **method_param)
        robust_errs.append(robust_err)
        errs.append(err)

    print('model robust errors (for robustly trained models, not valid for naturally trained models):')
    print(robust_errs)
    robust_errs = np.array(robust_errs)
    print('min: {:.4f}, max: {:.4f}, median: {:.4f}, mean: {:.4f}'.format(np.min(robust_errs), np.max(robust_errs), np.median(robust_errs), np.mean(robust_errs)))
    print('clean errors for models with min, max and median robust errors')
    i_min = np.argmin(robust_errs)
    i_max = np.argmax(robust_errs)
    i_median = np.argsort(robust_errs)[len(robust_errs) // 2]
    print('for min: {:.4f}, for max: {:.4f}, for median: {:.4f}'.format(errs[i_min], errs[i_max], errs[i_median]))
    print('model clean errors:')
    print(errs)
    print('min: {:.4f}, max: {:.4f}, median: {:.4f}, mean: {:.4f}'.format(np.min(errs), np.max(errs), np.median(errs), np.mean(errs))) 

def create_app():
    """Create Flask app."""
    config = load_config()

    app = Flask(__name__)
    app.config.from_object(config)

    # Proxy fix
    app.wsgi_app = ProxyFix(app.wsgi_app)

    # CSRF protect
    CsrfProtect(app)

    if app.debug or app.testing:
        DebugToolbarExtension(app)

        # Serve static files
        app.wsgi_app = SharedDataMiddleware(app.wsgi_app, {
            '/pages': os.path.join(app.config.get('PROJECT_PATH'), 'application/pages')
        })
    else:
        # Log errors to stderr in production mode
        app.logger.addHandler(logging.StreamHandler())
        app.logger.setLevel(logging.ERROR)

        # Enable Sentry
        if app.config.get('SENTRY_DSN'):
            from .utils.sentry import sentry

            sentry.init_app(app, dsn=app.config.get('SENTRY_DSN'))

        # Serve static files
        app.wsgi_app = SharedDataMiddleware(app.wsgi_app, {
            '/static': os.path.join(app.config.get('PROJECT_PATH'), 'output/static'),
            '/pkg': os.path.join(app.config.get('PROJECT_PATH'), 'output/pkg'),
            '/pages': os.path.join(app.config.get('PROJECT_PATH'), 'output/pages')
        })

    # Register components
    register_db(app)
    register_routes(app)
    register_jinja(app)
    register_error_handle(app)
    register_hooks(app)

    return app 

def create_app():
    """Create Flask app."""
    config = load_config()

    app = Flask(__name__)
    app.config.from_object(config)

    if not hasattr(app, 'production'):
        app.production = not app.debug and not app.testing

    # Proxy fix
    app.wsgi_app = ProxyFix(app.wsgi_app)

    # CSRF protect
    CsrfProtect(app)

    if app.debug or app.testing:
        DebugToolbarExtension(app)

        # Serve static files
        app.wsgi_app = SharedDataMiddleware(app.wsgi_app, {
            '/pages': os.path.join(app.config.get('PROJECT_PATH'), 'application/pages')
        })
    else:
        # Log errors to stderr in production mode
        app.logger.addHandler(logging.StreamHandler())
        app.logger.setLevel(logging.ERROR)

        # Enable Sentry
        if app.config.get('SENTRY_DSN'):
            from .utils.sentry import sentry

            sentry.init_app(app, dsn=app.config.get('SENTRY_DSN'))

        # Serve static files
        app.wsgi_app = SharedDataMiddleware(app.wsgi_app, {
            '/static': os.path.join(app.config.get('PROJECT_PATH'), 'output/static'),
            '/pkg': os.path.join(app.config.get('PROJECT_PATH'), 'output/pkg'),
            '/pages': os.path.join(app.config.get('PROJECT_PATH'), 'output/pages')
        })

    # Register components
    register_db(app)
    register_routes(app)
    register_jinja(app)
    register_error_handle(app)
    register_hooks(app)

    return app