Python fabric.api.settings() Examples

def build(name, ask=True, **kwargs):
    """
    Build the malicious mote to its target hardware.

    :param name: experiment name (or absolute path to experiment)
    :param ask: ask confirmation
    :param path: expanded path of the experiment (dynamically filled in through 'command' decorator with 'expand')
    :param kwargs: simulation keyword arguments (see the documentation for more information)
    """
    def is_device_present():
        with settings(hide(*HIDDEN_ALL), warn_only=True):
            return local("if [ -c /dev/ttyUSB0 ]; then echo 'ok'; else echo 'nok'; fi", capture=True) == 'ok'

    console = kwargs.get('console')
    counter, interval = 0.0, 0.5
    while not is_device_present():
        sleep(interval)
        counter += interval
        if counter % 5 == 0:
            logger.warning("Waiting for mote to be detected...")
        elif counter >= 120:
            logger.error("Something failed with the mote ; check that it mounts to /dev/ttyUSB0")
            return
    remake(name, build=True, **kwargs) if console is None else console.do_remake(name, build=True, **kwargs)
    return "Mote built on /dev/ttyUSB0" 

def __init__(self, name, cwd=None):
        """Create a tmux session.

        Args:
            name (str): name of the new session
            cwd (str): initial directory of the session

        Options used:
            -d: do not attach to the new session
            -s: specify a name for the session
        """
        self.name = name

        with settings(hide('warnings'), warn_only=True):
            result = local("tmux new -d -s {}".format(name))  # start tmux session

        if result.failed:
            raise TmuxSessionExists()

        if cwd is None:
            cwd = os.getcwd()

        # move to current directory
        self.run("cd {}".format(cwd)) 

def drop_db_connections(database_name):
    """
    looks up the db in local settings by its alias and drops any connections
    """

    drop_connections = """
        SELECT
            pg_terminate_backend(pid)
        FROM
            pg_stat_activity
        WHERE
            datname = '{db}'
        AND pid <> pg_backend_pid()
    """.format(db=database_name)

    psql(drop_connections) 

def __init__(self, name, cwd=None):
        """Create a tmux session.

        Args:
            name (str): name of the new session
            cwd (str): initial directory of the session

        Options used:
            -d: do not attach to the new session
            -s: specify a name for the session
        """
        self.name = name

        with settings(hide('warnings'), warn_only=True):
            result = local("tmux new -d -s {}".format(name))  # start tmux session

        if result.failed:
            raise TmuxSessionExists()

        if cwd is None:
            cwd = os.getcwd()

        # move to current directory
        self.run("cd {}".format(cwd)) 

def put_secure(user_group, mode, *args, **kwargs):
    missing_owner_code = 42
    user, group = user_group.split(":")

    files = put(*args, mode=mode, **kwargs)

    for file in files:
        with settings(warn_only=True):
            command = \
                "( getent passwd {user} >/dev/null || ( rm -f {file} ; " \
                "exit {missing_owner_code} ) ) && " \
                "chown {user_group} {file}".format(
                    user=user, file=file, user_group=user_group,
                    missing_owner_code=missing_owner_code)

            result = sudo(command)

            if result.return_code == missing_owner_code:
                abort("User %s does not exist. Make sure the Presto "
                      "server RPM is installed and try again" % (user,))
            elif result.failed:
                abort("Failed to chown file %s" % (file,)) 

def fix_solr():
    """ Fix solr
        work in progress
    """
    with cd('/var/tmp'):
        print blue('pulling new code...')
        sudo('/etc/init.d/jetty stop')
        sleep(5)
        # run('rm -rf /opt/deploy/solr/collection1')

        print blue('copying new code...')
        # run('mkdir -p /opt/deploy/solr/collection1')
        # run("cp -r oclapi/solr/collection1/conf /opt/deploy/solr/collection1")

    with cd("/opt/deploy/ocl_api/ocl"):
        # there is no need for this, settings.py.eploy is actually wrong?
        # run("cp settings.py.deploy settings.py")
        with prefix('source /opt/virtualenvs/ocl_api/bin/activate'):
            with prefix('export DJANGO_CONFIGURATION="Production"'):
                with prefix('export DJANGO_SECRET_KEY="blah"'):
                    # this is really slow because it pull down django-norel
                    run('./manage.py build_solr_schema > ' +
                        '/opt/deploy/solr/collection1/conf/schema.xml')
    sleep(5)
    sudo('/etc/init.d/jetty start') 

def deploy(target='dev', sha1=None):
    if sha1 is None:
        # get current working git sha1
        sha1 = local('git rev-parse HEAD', capture=True)
    # server code reset to current working sha1
    home_dir = '/home/pyconkr/{target}.pycon.kr/pyconkr-2016'.format(target=target)

    if target == 'dev':
        python_env = '/home/pyconkr/.pyenv/versions/pyconkr-2016-dev'
    else:
        python_env = '/home/pyconkr/.pyenv/versions/pyconkr-2016'

    with settings(cd(home_dir), shell_env(DJANGO_SETTINGS_MODULE='pyconkr.settings_prod')):
        sudo('git fetch --all -p', user='pyconkr')
        sudo('git reset --hard ' + sha1, user='pyconkr')
        sudo('bower install', user='pyconkr')
        sudo('%s/bin/pip install -r requirements.txt' % python_env, user='pyconkr')
        sudo('%s/bin/python manage.py compilemessages' % python_env, user='pyconkr')
        sudo('%s/bin/python manage.py migrate' % python_env, user='pyconkr')
        sudo('%s/bin/python manage.py collectstatic --noinput' % python_env, user='pyconkr')
        # worker reload
        run('echo r > /var/run/pyconkr-2016-%s.fifo' % target) 

def flatpages_mig(direction='www'):
    dev_env = '/home/pyconkr/.pyenv/versions/pyconkr-2016-dev/bin/python'
    www_env = '/home/pyconkr/.pyenv/versions/pyconkr-2016/bin/python'
    from_env, to_env = (dev_env, www_env) if direction=='www' else (www_env, dev_env)
    dev_dir = '/home/pyconkr/dev.pycon.kr/pyconkr-2016'
    www_dir = '/home/pyconkr/www.pycon.kr/pyconkr-2016'
    from_dir, to_dir = (dev_dir, www_dir) if direction=='www' else (www_dir, dev_dir)
    with settings(cd(from_dir),
            shell_env(DJANGO_SETTINGS_MODULE='pyconkr.settings_prod')
            ):
        sudo('{python} manage.py dumpdata --indent 2 flatpages -o {fixture_to}'.format(
            fixture_to=os.path.join(to_dir, 'pyconkr', 'fixtures', 'flatpages.json'),
            python=from_env))
    with settings(cd(to_dir),
            shell_env(DJANGO_SETTINGS_MODULE='pyconkr.settings_prod')
            ):
        sudo('{python} manage.py loaddata flatpages'.format(
             python=to_env)) 

def dev():
    """
    Put as the first task on the command line to select dev environment.
    For example: fab dev release_web_app
    Put in this task all the environment specific variables and settings.
    """
    env.hosts = ['dev.openconceptlab.com', ]
    env.user = 'deploy'
    env.web_domain = 'dev.openconceptlab.com'
    env.api_domain = 'api.dev.openconceptlab.com'
    env.OCL_API_TOKEN = os.environ.get('OCL_API_TOKEN')
    env.OCL_ANON_API_TOKEN = os.environ.get('OCL_ANON_API_TOKEN')
    env.random_string = _random_string(32)

    # which sites.json file to load the django site object from.
    env.site_spec = 'dev'

    env.AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
    env.AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
    env.AWS_STORAGE_BUCKET_NAME = 'ocl-source-export-development' 

def _install_cassandra_2_0():
     #install cassandra
    run('echo "deb http://debian.datastax.com/community stable main" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list')
    run('curl -L http://debian.datastax.com/debian/repo_key | sudo apt-key add -')
    sudo("apt-get update")
    sudo("sudo apt-get install --force-yes -y dsc20=2.0.12-1 cassandra=2.0.12 datastax-agent")
    with settings(warn_only=True):
        sudo("service cassandra stop")
    sudo("rm -rf /mnt/cassandra/data/system/*")
    sudo("rm -rf /mnt/cassandra/data/dse*")

    with settings(warn_only=True):
        sudo("mv /etc/security/limits.d/cassandra.conf /etc/security/limits.d/cassandra.conf.bak")

    run('echo "cassandra - memlock unlimited" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "cassandra - nofile 100000" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "cassandra - nproc 32768" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "cassandra - as unlimited" | sudo tee -a /etc/security/limits.d/cassandra.conf')

    # for Ubuntu
    run('echo "root - memlock unlimited" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "root - nofile 100000" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "root - nproc 32768" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "root - as unlimited" | sudo tee -a /etc/security/limits.d/cassandra.conf') 

def _install_cassandra_2_1():
     #install cassandra

    run('echo "deb http://debian.datastax.com/community stable main" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list')
    run('curl -L http://debian.datastax.com/debian/repo_key | sudo apt-key add -')
    sudo("apt-get update")
    sudo("sudo apt-get install --force-yes -y dsc21=2.1.9-1 cassandra=2.1.9 cassandra-tools=2.1.9 datastax-agent")
    with settings(warn_only=True):
        sudo("service cassandra stop")
    sudo("rm -rf /mnt/cassandra/data/system/*")
    sudo("rm -rf /mnt/cassandra/data/dse*")

    with settings(warn_only=True):
        sudo("mv /etc/security/limits.d/cassandra.conf /etc/security/limits.d/cassandra.conf.bak")

    run('echo "cassandra - memlock unlimited" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "cassandra - nofile 100000" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "cassandra - nproc 32768" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "cassandra - as unlimited" | sudo tee -a /etc/security/limits.d/cassandra.conf')

    # for Ubuntu
    run('echo "root - memlock unlimited" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "root - nofile 100000" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "root - nproc 32768" | sudo tee -a /etc/security/limits.d/cassandra.conf')
    run('echo "root - as unlimited" | sudo tee -a /etc/security/limits.d/cassandra.conf') 

def enable_dse(cluster_name, dse_url, dse_username, dse_password, dse_source_build_artifactory_url,
               dse_source_build_artifactory_username, dse_source_build_artifactory_password,
               dse_source_build_oauth_token):

    try:
        cluster = get_clusters(cluster_name, all_metadata=True)[cluster_name][0]
    except IndexError:
        raise ValueError("No cluster named {} found".format(cluster_name))

    cluster_ip = cluster['NetworkSettings']['IPAddress']
    with fab.settings(hosts=cluster_ip):
        fab_execute(fab_deploy.enable_dse, dse_url, dse_username, dse_password, dse_source_build_artifactory_url,
                    dse_source_build_artifactory_username, dse_source_build_artifactory_password,
                    dse_source_build_oauth_token)

    with fab.settings(hosts=cluster_ip, user="root"):
        fab_execute(tasks.restart_all_services) 

def setup_nginx():
    """Setup nginx.

    This can be re-run to update the application configuration via
    the ocl_nginx.conf.
    """
    with settings(warn_only=True):
        sudo('unlink /etc/nginx/sites-enabled/default')

    files.upload_template(_conf_path('ocl_nginx.conf'),
                          '/etc/nginx/sites-available/ocl', env, use_sudo=True)

    with settings(warn_only=True):
        sudo('ln -s /etc/nginx/sites-available/ocl /etc/nginx/sites-enabled/ocl')

    sudo('/etc/init.d/nginx restart') 

def bash(script, nodes=None, user=None):
    """Run a bash script on a set of nodes
    
    script - A bash script written as a string or list.
    nodes  - The set of nodes to run the command on. If None, all nodes of 
             the cluster will be used.
    user   - The user to run the command as. If None, the default user specified 
             in the cluster configuration    
    """ 
    if type(script) in (list, tuple):
        script = "\n".join(script)
    if nodes is None:
        nodes = common.fab.env.hosts
    if user is None:
        user = common.fab.env.user
    with common.fab.settings(user=user, hosts=nodes):
        return execute(common.bash, script) 

def deploy_code(host_string):
    max_retries = 20
    retries = 0
    log.debug("Waiting for instance to answer on ssh at {}".format(host_string))
    with settings(host_string="ec2-user@" + host_string):
        while True:
            try:
                fabtools.require.git.working_copy('https://github.com/creativecommons/open-ledger.git', branch=env.branch)
                with cd('open-ledger'):
                    run('virtualenv venv --python=python3 -q')
                    run('./venv/bin/pip install -r requirements.txt -q')
                    break
            except NetworkError:
                time.sleep(5)
                retries += 1
                log.debug("Retrying {} of {}...".format(retries, max_retries))
            if retries > max_retries:
                raise LoaderException("Timed out waiting for ssh") 

def build(self, tag, add_latest_tag=False, add_tags=None, raise_on_error=True, **kwargs):
        try:
            context = kwargs.pop('fileobj')
        except KeyError:
            raise ValueError("'fileobj' needs to be provided. Using 'path' is currently not implemented.")
        for a in ['custom_context', 'encoding']:
            kwargs.pop(a, None)

        with temp_dir() as remote_tmp:
            remote_fn = posixpath.join(remote_tmp, 'context')
            put(context, remote_fn)
            cmd_str = self._out.get_cmd('build', '- <', remote_fn, tag=tag, **kwargs)
            with settings(warn_only=not raise_on_error):
                res = self._call(cmd_str)
        if res:
            image_id = _find_image_id(res)
            if image_id:
                self.add_extra_tags(image_id, tag, add_tags, add_latest_tag)
                return image_id
        return None 

def generate_ssh_keys(self):
        for ssh_key_name, ssh_keygen_args in sorted(self.ssh_keys):
            if not exists(self.custom_template_path):
                os.mkdir(self.custom_template_path)
            ssh_key = join(self.custom_template_path, ssh_key_name)
            if exists(ssh_key):
                continue
            with settings(quiet(), warn_only=True):
                result = local(
                    "ssh-keygen %s -f %s -N ''" % (ssh_keygen_args, ssh_key),
                    capture=True)
                if result.failed:
                    print("Generation of %s with '%s' failed." % (
                        ssh_key_name, ssh_keygen_args))
                    continue
            with settings(quiet()):
                fingerprint = local(
                    "ssh-keygen -lf %s" % ssh_key, capture=True).split()[1]
            print("Generated %s with fingerprint %s." % (ssh_key_name, fingerprint)) 

def _setup_maven_authentication(config):
    dse_source_build_artifactory_username = config.get('dse_source_build_artifactory_username')
    dse_source_build_artifactory_password = config.get('dse_source_build_artifactory_password')
    dse_source_build_artifactory_url = config.get('dse_source_build_artifactory_url')

    maven_settings = "<settings><mirrors><mirror><id>artifactory</id><name>DataStax Maven repository</name>" \
                     "<url>{url}</url>" \
                     "<mirrorOf>central,java.net2,xerial,datanucleus,apache,datastax-public-snapshot,datastax-public-release,datastax-deps,datastax-release,datastax-snapshot</mirrorOf>" \
                     "</mirror></mirrors><servers><server><id>artifactory</id><username>{username}</username>" \
                     "<password>{password}</password></server></servers></settings>" \
        .format(username=dse_source_build_artifactory_username, password=dse_source_build_artifactory_password,
                url=dse_source_build_artifactory_url)

    fab.local('rm -rf ~/.m2/settings.xml')
    fab.local('mkdir -p ~/.m2')
    fab.local('echo "{maven_settings}" > ~/.m2/settings.xml'.format(maven_settings=maven_settings)) 

def get_client(self):
        if 'fabric_host' in self:
            with settings(host_string=self.fabric_host):
                return super(FabricClientConfiguration, self).get_client()
        return super(FabricClientConfiguration, self).get_client() 

def _random_string(number_chars):
    """ Generate a random string for settings.
    """
    return ''.join(random.sample(string.ascii_uppercase +
                                 string.ascii_lowercase + string.digits, number_chars)) 

def _restart_agent():
    with settings(warn_only=True):
        sudo("service datastax-agent restart")
        time.sleep(10) 

def setup_hosts_file(hosts):
    """Setup /etc/hosts
    
    hosts is a dictionary of hostname -> ip address
    """
    with fab.settings(user='root'):
        for host, ip in hosts.items():
            fab.run("echo '{ip} {host}' >> /etc/hosts".format(ip=ip, host=host), quiet=True) 

def update(silent=False, **kwargs):
    """
    Update Contiki-OS and RPL Attacks Framework.

    :param silent: run command silently
    :param kwargs: simulation keyword arguments (see the documentation for more information)
    """
    updated = False
    for folder, repository in zip([CONTIKI_FOLDER, FRAMEWORK_FOLDER], ["Contiki-OS", "RPL Attacks Framework"]):
        with hide(*HIDDEN_ALL):
            with lcd(folder):
                if "Could not resolve proxy" in local('git fetch --all', capture=True):
                    logger.error("Update failed ; please check your proxy settings")
                    break
                uptodate = "branch is up-to-date" in local('git checkout master', capture=True).strip().split('\n')[-1]
                if not uptodate:
                    req_exists = exists("requirements.txt")
                    if req_exists:
                        req_md5 = hash_file("requirements.txt")
                    logger.warn("You are about to loose any custom change made to {} ;".format(repository))
                    if silent or std_input("Proceed anyway ? (yes|no) [default: no] ", 'yellow') == 'yes':
                        local('git submodule update --init')
                        local('git fetch --all')
                        local('git reset --hard origin/master')
                        local('git pull')
                        if req_exists and hash_file("requirements.txt") != req_md5:
                            local('pip install -r requirements.txt')
                        updated = True
            if repository == "RPL Attacks Framework":
                remove_files(folder, "Vagrantfile")
                remove_folder(join(folder, "provisioning"))
            logger.debug(" > {} {}".format(repository, ["updated", "already up-to-date"][uptodate]))
    if updated:
        setup(silent)
        if not silent:
            logger.warn("Restarting the framework...")
            restart(PIDFILE) 

def test(**kwargs):
    """
    Run framework's tests.

    :param kwargs: simulation keyword arguments (see the documentation for more information)
    """
    with settings(warn_only=True):
        print(FRAMEWORK_FOLDER)
        with lcd(FRAMEWORK_FOLDER):
            local("python -m unittest -v tests") 

def console():
    """ Open framework's console. """
    with settings(remote_interrupt=False):
        local('python main.py') 

def cleanAll():
    with cd('loopix'):
        with cd('loopix'):
            run("rm *.log* *.bin example.db *.prv log.json", warn_only=True)
#
# @roles("mixnodes", "providers")
# @parallel
# def cleanServers():
#     with cd('loopix'):
#         with cd('loopix'):
#             run("rm *.log* *.bin example.db *.prv log.json", warn_only=True)
#
# @roles("clients")
# @parallel
# def cleanClients(num):
#     for i in range(int(num)):
#         with cd('client%d/loopix/loopix'%i):
#             run("rm *.log* *.bin example.db *.prv log.json", warn_only=True)
#
# @roles("mixnodes", "clients", "providers")
# @parallel
# def cleanSetup():
#     with settings(warn_only=True):
#         run("rm -rf loopix")
#         run("rm -rf client*")
#         run("rm -rf mixnode*")
#         run("rm -rf provider*")
#
# 

def deployMulti(num):
    with settings(warn_only=True):
        local("rm *.bin *.prv *.bi2 example.db")
    execute(deployMixnode)
    execute(deployProvider)
    execute(storeProvidersNames)
    execute(deployMultiClient,num)
    execute(readFiles)
    execute(loaddirServers)
    execute(loaddirClients,num) 

def deployAll():
    with settings(warn_only=True):
        local("rm *.bin *.bi2 example.db")
    execute(deployMixnode)
    execute(deployProvider)
    execute(storeProvidersNames)
    execute(deployClient)
    execute(readFiles)
    execute(loaddir) 

def install_cstar_perf_frontend():
    """Install the frontend

    This method assumes that Cassandra is already installed and running on the frontend node
    """
    fab.run("mkdir -p ~/git")
    fab.run("test ! -f ~/git/cstar_perf/frontend/setup.py && git -C ~/git clone http://github.com/datastax/cstar_perf.git; true")
    with fab.settings(user='root'):
        fab.run("pip install -e /home/cstar/git/cstar_perf/frontend") 

def set_permissions_on_spark_data_dir(nodes, spark_data_dir=os.path.join('/', 'var', 'lib', 'spark'), user='cstar'):
    with fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'), hosts=nodes):
        execute(fab.sudo, 'chmod -R 777 {spark_data}'.format(spark_data=spark_data_dir))
        execute(fab.sudo, 'chown {user}:{user} {spark_data}'.format(user=user, spark_data=spark_data_dir))