Python fabric.api.run() Examples

def keys():
    if "rsdir" not in env:
        secret = file("secret.key").read()
        public = rscoin.Key(secret, public=False)
        pid = b64encode(public.id())

        env["rsdir"] = {"special": pid, "directory": []}

    [_, host] = env.host_string.split("@")
    with cd('/home/ubuntu/projects/rscoin'):
        run('touch secret.key')
        run('rm secret.key')
        result = run('python derivekey.py --store')
        [_, key] = result.strip().split()
        
        kid = b64encode(rscoin.Key(b64decode(key)).id())
        env["rsdir"]["directory"] += [ [kid, host, 8080] ]
    

    from json import dumps
    file("directory.conf", "w").write(dumps(env["rsdir"])) 

def experiment1collect():        
        # run("ls experiment1/*")
    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('issue-times.txt', '%s/%s-issue-times.txt' % (env.expname, env.host))

    with lcd(env.expname):
        local("cat %s-issue-times.txt >> issue-times.txt" % env.host)

    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('r1-times.txt', '%s/%s-r1-times.txt' % (env.expname, env.host))
    
    with lcd(env.expname):
        local("cat %s-r1-times.txt >> r1-times.txt" % env.host)

    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('r2-times.txt', '%s/%s-r2-times.txt' % (env.expname, env.host))

    with lcd(env.expname):
        local("cat %s-r2-times.txt >> r2-times.txt" % env.host)

        # local("python exp1plot.py experiment1") 

def install(name):
    with act_virtualenv(name):
        run("pip install uwsgi")
        run("pip install flask==0.12.4")
        run("pip install flask-bootstrap")
        run("pip install flask-script")
        run("pip install flask-moment")
        run("pip install pyminizip==0.2.1")
        run("pip install flask_wtf")
        run("pip install flask_sqlalchemy")
        run("pip install sqlalchemy-migrate")
        run("pip install flask-mail")
        run("pip install flask-login")
        run("pip install flask-ldap-login")
        run("pip install python-ldap")
        run("pip install python-daemon") 

def _clone_code_from_local_path(from_path, to_path, run_as_sudo=True):
    cmd_fn = sudo if run_as_sudo else run
    git_local_submodule_config = [
        'git config {} {}'.format(submodule_config.key, submodule_config.value)
        for submodule_config in _get_local_submodule_urls(from_path)
    ]
    git_remote_submodule_config = [
        'git config {} {}'.format(submodule_config.key, submodule_config.value)
        for submodule_config in _get_remote_submodule_urls(from_path)
    ]

    with cd(from_path):
        cmd_fn('git clone {}/.git {}'.format(
            from_path,
            to_path
        ))

    with cd(to_path):
        cmd_fn('git config receive.denyCurrentBranch updateInstead')
        cmd_fn(' && '.join(git_local_submodule_config))
        cmd_fn('git submodule update --init --recursive')
        cmd_fn(' && '.join(git_remote_submodule_config)) 

def bash(script, nodes=None, user=None):
    """Run a bash script on a set of nodes
    
    script - A bash script written as a string or list.
    nodes  - The set of nodes to run the command on. If None, all nodes of 
             the cluster will be used.
    user   - The user to run the command as. If None, the default user specified 
             in the cluster configuration    
    """ 
    if type(script) in (list, tuple):
        script = "\n".join(script)
    if nodes is None:
        nodes = common.fab.env.hosts
    if user is None:
        user = common.fab.env.user
    with common.fab.settings(user=user, hosts=nodes):
        return execute(common.bash, script) 

def get_spark_cassandra_stress_command(script, node, master, stress_node=None):
    dse_bin = os.path.join(dse.get_dse_path(), 'bin')
    # see conversation on https://github.com/datastax/cstar_perf/pull/226 for why we pass SPARK_MASTER
    # tl;dr on DSE 4.7.x the dse script tries to call dsetool on the spark-cassandra-stress node
    # if SPARK_MASTER env var is not set and this results in a connection error trace as
    # we do not start DSE on the spark-cassandra-stress node
    spark_cassandra_stress_cmd_prefix = 'cd {spark_cass_stress_path}; ' \
                                        'PATH=$PATH:{dse_bin} ' \
                                        'JAVA_HOME={JAVA_HOME} ' \
                                        'DSE_HOME={dse_home} ' \
                                        'SPARK_MASTER={master} '.format(spark_cass_stress_path=get_spark_cassandra_stress_path(stress_node=stress_node),
                                                                        dse_bin=dse_bin,
                                                                        JAVA_HOME=JAVA_HOME,
                                                                        dse_home=dse.get_dse_path(),
                                                                        master=master)

    spark_cass_connection_host_arg = ' --conf spark.cassandra.connection.host={node}'.format(node=node)
    spark_cassandra_run_cmd = './run.sh dse {script} {master} {connection_host}'.format(script=script,
                                                                                        master=master,
                                                                                        connection_host=spark_cass_connection_host_arg)
    cmd = spark_cassandra_stress_cmd_prefix + ' ' + spark_cassandra_run_cmd

    return cmd 

def engine_start(http_host, http_port):
    package = env.package

    command = (
        "workon {package}_env &&"
        " (marvin engine-httpserver"
        " -h {http_host}"
        " -p {http_port}"
        " -e {executor}"
        " 1> /var/log/marvin/engines/{package}.out"
        " 2> /var/log/marvin/engines/{package}.err"
        " & echo $! > /var/run/marvin/engines/{package}.pid)"
    ).format(
        package=package,
        http_host=http_host,
        http_port=http_port,
        executor=env.marvin_engine_executor_path
    )

    with cd("/opt/marvin/engines/{package}/current".format(package=package)):
        run(command, pty=False) 

def postgresql():
    run('service postgresql status')
    run('service pgbouncer status') 

def npm_install():
    with cd("/srv/calthorpe/urbanfootprint/websockets"):
        # Symlink issues with Windows+Virtualbox so we add '--no-bin-link'
        # Similar to: https://github.com/npm/npm/issues/7094
        run("/usr/bin/npm install --no-bin-links .") 

def stop_perf_agent():
    logger.info("Stopping Flamegraph perf agent")
    common_module.run_python_script(
        'utils',
        'find_and_kill_process',
        '"{}", {}'.format("[^/]perf record -F", 'child_process=True')
    )

    flamegraph_directory = get_flamegraph_directory()
    fab.run('sudo chmod o+r {}'.format(os.path.join(flamegraph_directory, '*'))) 

def create_offline_dir():
    run('mkdir -p {}'.format(env.offline_code_dir)) 

def sync_offline_from_control():
    for host in _hosts_in_roles(ROLES_ALL_SRC, exclude_roles=ROLES_DEPLOY):
        run("rsync -rvz --exclude 'commcare-hq/*' {} {}".format(
            env.offline_code_dir,
            '{}@{}:{}'.format(env.user, host, env.offline_releases)
        )) 

def restart_all_services():
    fab.run("test -f /supervisord.conf && supervisorctl -c /supervisord.conf reread")
    fab.run("test -f /supervisord.conf && supervisorctl -c /supervisord.conf update")
    fab.run("test -f /supervisord.conf && supervisorctl -c /supervisord.conf restart all") 

def client():
    client = os.environ.get('CLIENT')
    client_path = "footprint/client/configuration/{client}".format(client=client)
    sproutcore_path = "sproutcore/frameworks/footprint/resources/images"
    with cd('/srv/calthorpe/urbanfootprint'):
        run('if [ -e {client_path}/login_page_logo.png ]; then cp -f {client_path}/login_page_logo.png {sproutcore_path}/client_login_page_logo.png; fi'.format(client_path=client_path, sproutcore_path=sproutcore_path))
        run('if [ -e {client_path}/login_page_logo_lower.png ]; then cp -f {client_path}/login_page_logo.png {sproutcore_path}/client_login_page_logo_lower.png; fi'.format(client_path=client_path, sproutcore_path=sproutcore_path)) 

def get_commit(build_type='prod'):
    with cd(get_django_setting(build_type, 'ROOT_PATH')):
        with hide('output'):
            commit = run("git rev-list HEAD --max-count=1 --pretty --oneline").stdout
    print str(commit)
    return commit 

def check_docker_version(expected_version='1.6.0'):
    version_cmd = shlex.split("docker --version")
    try:
        p = subprocess.Popen(version_cmd, stdout=subprocess.PIPE)
        version_string = p.communicate()[0]
    except OSError:
        raise AssertionError('Failed to run docker, it may not be installed?')
    m = re.match('Docker version ([^,]+), .*', version_string)
    if m:
        version = m.groups()[0]
        if LooseVersion(version) < expected_version:
            raise AssertionError(
                'Found docker version {}. This tool requires version {}+'.format(
                    version, expected_version)) 

def cqlsh(script, node):
    """Run a cqlsh script on a node"""
    global cqlsh_path
    script = script.replace('\n', ' ')
    cmd = '{cqlsh_path} --no-color {host} -e "{script}"'.format(cqlsh_path=cqlsh_path, host=node, script=script)

    with common.fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'), hosts=node):
        return execute(fab.run, cmd)[node] 

def dse_cmd(node, options):
    cmd = "JAVA_HOME={java_home} {dse_cmd} {options}".format(java_home=JAVA_HOME,
                                                             dse_cmd=os.path.join(dse.get_bin_path(), 'dse'),
                                                             options=options)
    with common.fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'), hosts=node, warn_only=True):
        return execute(fab.run, cmd)[node] 

def elasticsearch():
    global ELASTICSEARCH_CHECKED
    if not ELASTICSEARCH_CHECKED:
        run("curl -XGET 'http://%s:9200/_cluster/health?pretty=true'" % env.host)
        ELASTICSEARCH_CHECKED = True
    run('service elasticsearch status') 

def ping():
    with hide('running', 'output'):
        hostname = run('hostname')
        vmballoon = run('vmware-toolbox-cmd stat balloon')
        vmswap = run('vmware-toolbox-cmd stat swap')
        uptime = run('uptime')
        free = run('free -h')
        print("===== Hello from %s =====" % hostname)
        print("[%s] %s" % (hostname, uptime))
        print("[%s] %s" % (hostname, free))
        print("[%s] vmware balloon: %s" % (hostname, vmballoon))
        print("[%s] vmware swap: %s" % (hostname, vmswap))
    #with hide('running'): 

def _run_docker_component(component):
  with cd("$HOME"):
    with cd("storm-docker"):
      run("./destroy-storm.sh {}".format(component), warn_only=True)
      run("./start-storm.sh {}".format(component)) 

def fetch_repo():
    with cd(code_dir):
        with settings(warn_only=True):
            run("source venv/bin/activate")
            # run("pip install -r requirements.txt")
            run("git pull") 

def deploy_api():
    fetch_repo()
    run("tmux a -t dive")
    run("pkill gunicorn")
    run("source venv/bin/activate")
    run("sh run_server.sh") 

def run(*args, **kwargs):
    ret = yield retry(_run, *args, **kwargs, quiet=FABRIC_QUIET)
    return ret 

def _clean_up_cdc_directories():
    if config.get('cdc_overflow_directory'):
        fab.run('rm -rf {overflow_dir}/*'.format(overflow_dir=config['cdc_overflow_directory']))
    if config.get('cdc_directory'):
        fab.run('rm -rf {cdc_dir}/*'.format(cdc_dir=config['cdc_directory'])) 

def experiment1actual():
    with cd('/home/ubuntu/projects/rscoin'):
        run("./rsc.py --play payments.txt-r2 > %s/r2-times.txt" % env.expname) 

def experiment1pre():
    with cd('/home/ubuntu/projects/rscoin'):
        run("./rsc.py --play payments.txt-r1 --conn 30 > %s/r1-times.txt" % env.expname) 

def experiment1run():
    # local('sudo sysctl -w net.ipv4.ip_local_port_range="500   65535"')
    # local("sudo echo 20000500 > /proc/sys/fs/nr_open")
    # local('sudo sh -c "ulimit -n 1048576"')
    with cd('/home/ubuntu/projects/rscoin'):
        run("python simscript.py %s payments.txt" % env.messages)
        run("rm -rf %s" % env.expname)
        run("mkdir %s" % env.expname)
        run("./rsc.py --play payments.txt-issue > %s/issue-times.txt" % env.expname)
        # run("./rsc.py --play payments.txt-r1 > experiment1/r1-times.txt") 

def destroy(leave_data=False, kill_delay=0):
    """Uninstall Cassandra and clean up data and logs"""
    # We used to have a better pattern match for the Cassandra
    # process, but it got fragile if you put too many JVM params.
    if leave_data:
        fab.run('JAVA_HOME={java_home} {nodetool_cmd} drain'.format(java_home=config['java_home'], nodetool_cmd=_nodetool_cmd()), quiet=True)
        fab.run('rm -rf {commitlog}/*'.format(commitlog=config['commitlog_directory']))
        _clean_up_cdc_directories()

    if kill_delay:
        fab.run('killall java', quiet=True)
        time.sleep(kill_delay)   # kill delay waiting the jvm to exit, profiling stuff require some time to be dumped
    fab.run('killall -9 java', quiet=True)
    fab.run('pkill -f "python.*fincore_capture"', quiet=True)
    fab.run('rm -rf fab/cassandra')
    fab.run('rm -rf fab/dse')
    fab.run('rm -rf fab/scripts')
    fab.run('rm -f {startup_log}'.format(startup_log=CASSANDRA_STARTUP_LOG))

    # Ensure directory configurations look sane
    assert type(config['data_file_directories']) == list
    for t in [config['saved_caches_directory'], config['commitlog_directory'],
              config['flush_directory'], config['log_dir']] + config['data_file_directories']:
        assert type(t) in (str, unicode) and len(t) > 1, '{t} doesn\'t look like a directory'.format(t=t)

    if not leave_data:
        for d in config['data_file_directories']:
            fab.run('rm -rf {data}/*'.format(data=d))
        fab.run('rm -rf {saved_caches_directory}/*'.format(saved_caches_directory=config['saved_caches_directory']))
        fab.run('rm -rf {commitlog}/*'.format(commitlog=config['commitlog_directory']))
        fab.run('rm -rf {flushdir}/*'.format(flushdir=config['flush_directory']))
        if config.get('hints_directory'):
            fab.run('rm -rf {hints_directory}/*'.format(hints_directory=config.get('hints_directory')))
        _clean_up_cdc_directories()
    fab.run('rm -rf {log_dir}/*'.format(log_dir=config['log_dir']))
    fab.run('rm -f /tmp/fincore.stats.log') 

def download_and_build_spark_cassandra_stress(stress_node=None):
    dse_home = 'DSE_HOME={dse_path}'.format(dse_path=dse.get_dse_path())
    dse_resources = 'DSE_RESOURCES={dse_resources_path}'.format(dse_resources_path=os.path.join(dse.get_dse_path(), 'resources'))
    spark_cassandra_stress_git = 'https://github.com/datastax/spark-cassandra-stress.git'
    git_clone_spark_cass_stress_command = 'git clone -b master --single-branch ' \
                                          '{spark_cass_stress_git} ' \
                                          '{spark_cass_stress_path}'.format(spark_cass_stress_git=spark_cassandra_stress_git,
                                                                            spark_cass_stress_path=get_spark_cassandra_stress_path(stress_node=stress_node))
    build_command = './gradlew jar -Pagainst=dse;'
    full_build_command = 'cd {spark_cass_stress_path}; TERM=dumb {dse_home} {dse_resources} {build_cmd}'.format(
        spark_cass_stress_path=get_spark_cassandra_stress_path(),
        dse_home=dse_home,
        dse_resources=dse_resources,
        build_cmd=build_command
    )

    if stress_node:
        with common.fab.settings(hosts=stress_node):
            execute(fab.run, 'rm -rf {spark_cass_stress_path}'.format(spark_cass_stress_path=get_spark_cassandra_stress_path(stress_node=stress_node)))
            execute(fab.run, git_clone_spark_cass_stress_command)
            execute(fab.run, full_build_command)
    else:
        shutil.rmtree(get_spark_cassandra_stress_path(), ignore_errors=True)
        logger.info('Installing Spark-Cassandra-Stress from {spark_cass_stress_git}'.format(spark_cass_stress_git=spark_cassandra_stress_git))
        proc = subprocess.Popen(git_clone_spark_cass_stress_command, shell=True)
        proc.wait()
        assert proc.returncode == 0, 'Installing Spark-Cassandra-Stress from {spark_cass_stress_git} ' \
                                     'did not complete successfully'.format(spark_cass_stress_git=spark_cassandra_stress_git)
        logger.info('Building Spark-Cassandra-Stress using {full_build_command}'.format(full_build_command=full_build_command))
        proc = subprocess.Popen(full_build_command, shell=True)
        proc.wait()
        assert proc.returncode == 0, 'Building Spark-Cassandra-Stress using {full_build_command} ' \
                                     'did not complete successfully'.format(full_build_command=full_build_command)