Python scp.SCPClient() Examples

def fetch_remote_crashes(self):
        """
        some exception handling code is taken from https://www.programcreek.com/python/example/105570/scp.SCPClient
        """
        try:
            ssh = SSHClient()
            ssh.load_system_host_keys()
            ssh.connect(hostname=config.remote_system_ip)
            self.copy_crashes_dir_with_scp(ssh)
        except AuthenticationException:
            print("Authentication failed, please verify your credentials: %s")
        except SSHException as sshException:
            print("Unable to establish SSH connection: %s" % sshException)
        except BadHostKeyException as badHostKeyException:
            print("Unable to verify server's host key: %s" % badHostKeyException)
        finally:
            ssh.close() 

def _scp_put(self, hostname=None, username=None, password=None, srcfile=None, destfile=None):
        """SCP Put file

        :param hostname: FQDN/IP
        :param username: Username
        :param password: Password
        :param srcfile: Source file
        :param destfile: Destination file
        """
        self.ssh_client.connect(hostname=hostname,
                                username=username,
                                password=password)
        scp_client = scp.SCPClient(self.ssh_client.get_transport())
        if '*' in srcfile:
            listing = glob.glob(srcfile)
            if len(listing) == 0:
                raise Exception("No file found: " + srcfile)
            srcfile = listing
        scp_client.put(srcfile, destfile, recursive=True)
        scp_client.close() 

def sftp_put(self, src, dest):
        """ Puts a file to the remote server

            Arguments:
            - src: Source local file to copy.
            - dest: Destination path in the remote server.
        """
        client, proxy = self.connect()
        transport = client.get_transport()
        try:
            sftp = paramiko.SFTPClient.from_transport(transport)
            if not transport.active:
                sftp = scp.SCPClient(transport)
        except Exception:
            # in case of failure try to use scp
            sftp = scp.SCPClient(transport)
        sftp.put(src, dest)
        sftp.close()
        if proxy:
            proxy.close()
        transport.close() 

def sftp_get(self, src, dest):
        """ Gets a file from the remote server

            Arguments:
            - src: Source file in the remote server.
            - dest: Local destination path to copy.
        """
        client, proxy = self.connect()
        transport = client.get_transport()

        try:
            sftp = paramiko.SFTPClient.from_transport(transport)
            if not transport.active:
                sftp = scp.SCPClient(transport)
        except Exception:
            # in case of failure try to use scp
            sftp = scp.SCPClient(transport)

        sftp.get(src, dest)
        sftp.close()
        if proxy:
            proxy.close()
        transport.close() 

def disconnect(self):
        """ Close the connection(s) to the device.

        Purpose: Closes the current connection(s) to the device, no matter
               | what types exist.

        @returns: None
        @rtype: None
        """
        if self._shell:
            self._shell.close()
            self._shell = ""
        if isinstance(self._session, manager.Manager):
            self._session.close_session()
        elif isinstance(self._session, paramiko.client.SSHClient):
            self._session.close()
            self._session = ""
        elif isinstance(self._session, SCPClient):
            self._session.close()
            self._session = ""
            self._scp = "" 

def transfer_file(self, ssh_client, files):
        if self.protocol == "sftp":
            with SFTPClient.from_transport(
                ssh_client.get_transport(),
                window_size=self.window_size,
                max_packet_size=self.max_transfer_size,
            ) as sftp:
                sftp.get_channel().settimeout(self.timeout)
                for source, destination in files:
                    getattr(sftp, self.direction)(source, destination)
        else:
            with SCPClient(
                ssh_client.get_transport(), socket_timeout=self.timeout
            ) as scp:
                for source, destination in files:
                    getattr(scp, self.direction)(source, destination) 

def conn(self):
        #print("Opening SSH connection to target...")
        self.ssh = paramiko.SSHClient()#use ssh.exec_command("") to perform an action.
        self.ssh.load_system_host_keys()
        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        self.ssh.connect(self.hostname, port=self.port, username=self.uname, password=self.pword)
        self.scp = SCPClient(self.ssh.get_transport())#don't call this, but use the above function instead.
        self.is_open = True
    #TODO: fix rm -rf bug 

def loginandcopy(hostname,uname,pwd,sfile,tfile):
     try:
        log.info("Establishing ssh connection")
        client = getsshClient()
        client.load_system_host_keys()
        client.connect(hostname)#,username=uname)#,password=pwd)
     except paramiko.AuthenticationException:
        print("Authentication failed, please verify your credentials: %s")
     except paramiko.SSHException as sshException:
        print("Unable to establish SSH connection: %s" % sshException)
     except paramiko.BadHostKeyException as badHostKeyException:
        print("Unable to verify server's host key: %s" % badHostKeyException)
     except Exception as e:
        print(e.args)
     try:
        log.info("Getting SCP Client")
        scpclient = scp.SCPClient(client.get_transport())
        log.info(scpclient)
        log.info("Hostname: %s", hostname)
        log.info("source file: %s", sfile)
        log.info("target file: %s", tfile)
        scpclient.put(sfile,tfile)
     except scp.SCPException as e:
        print("Operation error: %s", e) 

def loginandcopydir(hostname,uname,pwd,sfile,tfile,recursive,preserve_times):
     try:
        log.info("Establishing ssh connection")
        client = getsshClient()
        client.load_system_host_keys()
        client.connect(hostname) #,username=uname)#,password=pwd)
     except paramiko.AuthenticationException:
        print("Authentication failed, please verify your credentials: %s")
     except paramiko.SSHException as sshException:
        print("Unable to establish SSH connection: %s" % sshException)
     except paramiko.BadHostKeyException as badHostKeyException:
        print("Unable to verify server's host key: %s" % badHostKeyException)
     except Exception as e:
        print(e.args)
     try:
        scpclient = scp.SCPClient(client.get_transport())
        scpclient.put(sfile,tfile,recursive,preserve_times)
     except scp.SCPException as e:
        print("Operation error: %s", e)

# Deprecated 

def upload_and_create_directory(vm_name, hdd_size, source_file):
    commands = ["mkdir /vmfs/volumes/datastore1/{0}".format(vm_name),
                "vmkfstools -c {0}g -a lsilogic -d zeroedthick /vmfs/volumes/datastore1/{1}/{1}.vmdk".format(hdd_size,
                                                                                                             vm_name), ]
    register_command = "vim-cmd solo/registervm /vmfs/volumes/datastore1/{0}/{0}.vmx".format(vm_name)

    ipaddr = "10.10.10.115"
    username = "root"
    password = "access123"

    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

    ssh.connect(ipaddr, username=username, password=password, look_for_keys=False, allow_agent=False)

    for cmd in commands:
        try:
            stdin, stdout, stderr = ssh.exec_command(cmd)
            print " DEBUG: ... Executing the command on ESXI server".format(str(stdout.readlines()))

        except Exception as e:
            print e
            pass
            print " DEBUG: **ERR....unable to execute command"
        time.sleep(2)
    with SCPClient(ssh.get_transport()) as scp:
        print(" DEBUG: ... Uploading file to the datastore")
        scp.put(source_file, remote_path='/vmfs/volumes/datastore1/{0}'.format(vm_name))
        print(" DEBUG: ... Register the virtual machine {}".format(vm_name))
        ssh.exec_command(register_command)

    ssh.close() 

def copy_crashes_dir_with_scp(self, ssh):
        parent_dir_of_crashes_dir = os.path.dirname(config.crashes_dir)
        try:
            scp = SCPClient(ssh.get_transport())
            scp.get(remote_path=config.remote_crashes_dir, local_path=parent_dir_of_crashes_dir, recursive=True,
                    preserve_times=True)
            print("successfully fetched!!")
        except SCPException as e:
            print("Operation error: %s" % e)
        except SocketTimeout:
            """
            the fetcher will need multiple attempts if the ssh connection is bad and/or the copy dir is big
            """
            print('SocketTimeout')
        except PipeTimeout as pipetimeout:
            print("timeout was reached on a read from a buffered Pipe: %s" % pipetimeout)
        finally:
            scp.close() 

def sftp_get_files(self, src, dest):
        """ Gets a list of files from the remote server

            Arguments:
            - src: A list with the source files in the remote server.
            - dest: A list with the local destination paths to copy.
        """
        client, proxy = self.connect()
        transport = client.get_transport()
        try:
            sftp = paramiko.SFTPClient.from_transport(transport)
            if not transport.active:
                sftp = scp.SCPClient(transport)
        except Exception:
            # in case of failure try to use scp
            sftp = scp.SCPClient(transport)

        for file0, file1 in zip(src, dest):
            sftp.get(file0, file1)
        sftp.close()
        if proxy:
            proxy.close()
        transport.close() 

def sftp_put_files(self, files):
        """ Puts a list of files to the remote server

            Arguments:
            - files: A tuple where the first elements is the local source file to copy and the second
                     element the destination paths in the remote server.
        """
        client, proxy = self.connect()
        transport = client.get_transport()
        try:
            sftp = paramiko.SFTPClient.from_transport(transport)
            if not transport.active:
                sftp = scp.SCPClient(transport)
        except Exception:
            # in case of failure try to use scp
            sftp = scp.SCPClient(transport)

        for src, dest in files:
            sftp.put(src, dest)
        sftp.close()
        if proxy:
            proxy.close()
        transport.close() 

def provision_file_put(ssh_client, local_file, remote_file):
    print "put", local_file, remote_file
    scp_client = scp.SCPClient(ssh_client.get_transport())
    scp_client.put(local_file, remote_file)
    scp_client.close() 

def upload_migration_xml(api, workflow_dir, filename):
    hostname = urlparse(api.get_url()).hostname
    print('Hostname: %s' % hostname)
    ssh = SSHClient()
    ssh.set_missing_host_key_policy(AutoAddPolicy())
    ssh.connect(hostname, username='root', password='root')
    
    scp = SCPClient(ssh.get_transport())
    scp.put(workflow_dir + '/' + filename, '/data/migration/incoming/') 

def conn(self):
        #print("Opening SSH connection to target...")
        self.ssh = paramiko.SSHClient()  # use ssh.exec_command("") to perform an action.
        self.ssh.load_system_host_keys()
        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        self.ssh.connect(self.hostname, port=self.port, username=self.uname, password=self.pword)
        self.scp = SCPClient(self.ssh.get_transport())#don't call this, but use the above function instead.
        self.is_open = True 

def startscp(self):
        self.scp = SCPClient(self.ssh.get_transport()) 

def __insert_file(self):
        print '[+] uploading file...'
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        # get config settings
        host = self.__config.get('scp', 'scp.host')
        user = self.__config.get('scp', 'scp.user')
        password = self.__config.get('scp', 'scp.password')
        timeout = self.__config.get('scp', 'scp.timeout')
        self.info['upload_path'] = self.__config.get('scp', 'scp.path')

        ssh.connect(host, username=user, password=password)
        scpclient = SCPClient(ssh.get_transport(), socket_timeout=float(timeout))
        scpclient.put(self.info['path'], self.info['upload_path'] + self.info['name']) 

def _prepare_connection(**kwargs):
    '''
    Prepare the underlying SSH connection with the remote target.
    '''
    paramiko_kwargs, scp_kwargs = _select_kwargs(**kwargs)
    ssh = paramiko.SSHClient()
    if paramiko_kwargs.pop('auto_add_policy', False):
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(**paramiko_kwargs)
    scp_client = scp.SCPClient(ssh.get_transport(),
                               **scp_kwargs)
    return scp_client 

def _select_kwargs(**kwargs):
    paramiko_kwargs = {}
    scp_kwargs = {}
    PARAMIKO_KWARGS, _, _, _ = inspect.getargspec(paramiko.SSHClient.connect)
    PARAMIKO_KWARGS.pop(0)  # strip self
    PARAMIKO_KWARGS.append('auto_add_policy')
    SCP_KWARGS, _, _, _ = inspect.getargspec(scp.SCPClient.__init__)
    SCP_KWARGS.pop(0)  # strip self
    SCP_KWARGS.pop(0)  # strip transport arg (it is passed in _prepare_connection)
    for karg, warg in six.iteritems(kwargs):
        if karg in PARAMIKO_KWARGS and warg is not None:
            paramiko_kwargs[karg] = warg
        if karg in SCP_KWARGS and warg is not None:
            scp_kwargs[karg] = warg
    return paramiko_kwargs, scp_kwargs 

def transfer_file(self, dest):
        """Begin to transfer file by scp"""

        if not self.local_file_exists():
            self.module.fail_json(
                msg='Could not transfer file. Local file doesn\'t exist.')

        if not self.enough_space():
            self.module.fail_json(
                msg='Could not transfer file. Not enough space on device.')

        hostname = self.module.params['provider']['host']
        username = self.module.params['provider']['username']
        password = self.module.params['provider']['password']
        port = self.module.params['provider']['port']

        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(hostname=hostname, username=username, password=password, port=port)
        full_remote_path = '{}{}'.format(self.file_system, dest)
        scp = SCPClient(ssh.get_transport())
        try:
            scp.put(self.local_file, full_remote_path)
        except:
            time.sleep(10)
            file_exists, temp_size = self.remote_file_exists(
                dest, self.file_system)
            file_size = os.path.getsize(self.local_file)
            if file_exists and int(temp_size) == int(file_size):
                pass
            else:
                scp.close()
                self.module.fail_json(msg='Could not transfer file. There was an error '
                                      'during transfer. Please make sure the format of '
                                      'input parameters is right.')

        scp.close()
        return True 

def _scp_get(self, hostname=None, username=None, password=None, srcfile=None, destfile=None):
        """SCP Get file

        :param hostname: FQDN/IP
        :param username: Username
        :param password: Password
        :param srcfile: Source file
        :param destfile: Destination file
        """
        self.ssh_client.connect(hostname=hostname,
                                username=username,
                                password=password)
        scp_client = scp.SCPClient(self.ssh_client.get_transport(), sanitize=lambda x: x)
        scp_client.get(srcfile, destfile, recursive=True)
        scp_client.close() 

def Scp(self,src,dst):
        scp = SCPClient(self._ssh.get_transport())
        stderr = scp.put(src,dst,recursive=True)
        if stderr:
            return stderr 

def conn(self):
        #print("Opening SSH connection to target...")
        self.ssh = paramiko.SSHClient()#use ssh.exec_command("") to perform an action.
        self.ssh.load_system_host_keys()
        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        self.ssh.connect(self.hostname, port=self.port, username=self.uname, password=self.pword)
        self.scp = SCPClient(self.ssh.get_transport())#don't call this, but use the above function instead.
        self.is_open = True
    #TODO: fix rm -rf bug 

def request_scp_transfer(self):
        """ Begins SCP file transfer with progress """
        self.ensure_scp()
        self._init_transfer()
        source = self.upgrade_image_src_path
        dest = self.upgrade_image_dest_path
        ssh_connect_params = self.ft.ssh_ctl_chan._connect_params_dict()
        self.ft.scp_conn = self.ft.ssh_ctl_chan._build_ssh_client()
        self.ft.scp_conn.connect(**ssh_connect_params)
        with tqdm(unit="b", unit_scale=True, ascii=True) as t:
            self.progress = self._scp_tqdm(t)
            self.ft.scp_client = scp.SCPClient(
                self.ft.scp_conn.get_transport(), progress=self.progress
            )
            self.ft.scp_client.put(source, dest) 

def update_prometheus_config(self, base_file):
        log.info("Updating prometheus configuration")
        aggregator_host_port = os.getenv('aggregator_endpoint').split('/')[2]
        aggregator_host = aggregator_host_port.split(':')[0]
        #paramiko_helper.loginandcopy(aggregator_host, None, None, base_file,'/etc/prometheus/')#, True, True)
        sshclient = paramiko.SSHClient()
        sshclient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        sshclient.load_system_host_keys()
        log.info("Aggregator Host: %s", aggregator_host)
        # Currently supporting username/password and key_file based authentication. ToDO add pkey parameter support in connect method
        sshclient.connect(aggregator_host, username=CONF.gapi.username, password=CONF.gapi.password, key_filename=CONF.gapi.key_filename)
        scpclient = scp.SCPClient(sshclient.get_transport())
        scpclient.put(base_file, '/etc/prometheus') 

def scp_push_file(client, client_ftp, transfer_file, local_dir, remote_dir,
    CLOBBER=False, VERBOSE=False, LIST=False, MODE=0o775):
    #-- local and remote versions of file
    local_file = os.path.join(local_dir,transfer_file)
    remote_file = posixpath.join(remote_dir,transfer_file)
    #-- check if local file is newer than the remote file
    TEST = False
    OVERWRITE = 'clobber'
    if (transfer_file in client_ftp.listdir(remote_dir)):
        local_mtime = os.stat(local_file).st_mtime
        remote_mtime = client_ftp.stat(remote_file).st_mtime
        #-- if local file is newer: overwrite the remote file
        if (even(local_mtime) > even(remote_mtime)):
            TEST = True
            OVERWRITE = 'overwrite'
    else:
        TEST = True
        OVERWRITE = 'new'
    #-- if file does not exist remotely, is to be overwritten, or CLOBBER is set
    if TEST or CLOBBER:
        if VERBOSE or LIST:
            print('{0} --> '.format(local_file))
            print('\t{0} ({1})\n'.format(remote_file,OVERWRITE))
        #-- if not only listing files
        if not LIST:
            #-- copy local files to remote server
            with scp.SCPClient(client.get_transport(), socket_timeout=20) as s:
                s.put(local_file, remote_file, preserve_times=True)
            #-- change the permissions level of the transported file to MODE
            client_ftp.chmod(remote_file, MODE)

#-- PURPOSE: pull file from a remote host checking if file exists locally
#-- and if the remote file is newer than the local file (reprocessed)
#-- set the permissions mode of the local transferred file to MODE 

def provision_file_get(ssh_client, remote_file, local_file):
    print "get", remote_file, local_file
    scp_client = scp.SCPClient(ssh_client.get_transport())
    scp_client.get(remote_file, local_file)
    scp_client.close() 

def scp_pull_file(client, client_ftp, transfer_file, local_dir, remote_dir,
    CLOBBER=False, VERBOSE=False, LIST=False, MODE=0o775):
    #-- local and remote versions of file
    local_file = os.path.join(local_dir,transfer_file)
    remote_file = posixpath.join(remote_dir,transfer_file)
    #-- check if remote file is newer than the local file
    TEST = False
    OVERWRITE = 'clobber'
    if os.access(local_file, os.F_OK):
        local_mtime = os.stat(local_file).st_mtime
        remote_mtime = client_ftp.stat(remote_file).st_mtime
        #-- if remote file is newer: overwrite the local file
        if (even(remote_mtime) > even(local_mtime)):
            TEST = True
            OVERWRITE = 'overwrite'
    else:
        TEST = True
        OVERWRITE = 'new'
    #-- if file does not exist locally, is to be overwritten, or CLOBBER is set
    if TEST or CLOBBER:
        if VERBOSE or LIST:
            print('{0} --> '.format(remote_file))
            print('\t{0} ({1})\n'.format(local_file,OVERWRITE))
        #-- if not only listing files
        if not LIST:
            #-- copy local files from remote server
            with scp.SCPClient(client.get_transport(), socket_timeout=20) as s:
                s.get(remote_file, local_path=local_file, preserve_times=True)
            #-- change the permissions level of the transported file to MODE
            os.chmod(local_file, MODE)

#-- PURPOSE: rounds a number to an even number less than or equal to original 

def copy_file(
        host,
        file_path,
        remote_path='.',
        username=None,
        key_path=None,
        action='put'
):
    """ Copy a file via SCP, proxied through the mesos master

        :param host: host or IP of the machine to execute the command on
        :type host: str
        :param file_path: the local path to the file to be copied
        :type file_path: str
        :param remote_path: the remote path to copy the file to
        :type remote_path: str
        :param username: SSH username
        :type username: str
        :param key_path: path to the SSH private key to use for SSH authentication
        :type key_path: str

        :return: True if successful, False otherwise
        :rtype: bool
    """

    if not username:
        username = shakedown.cli.ssh_user

    if not key_path:
        key_path = shakedown.cli.ssh_key_file

    key = validate_key(key_path)

    transport = get_transport(host, username, key)
    transport = start_transport(transport, username, key)

    if transport.is_authenticated():
        start = time.time()

        channel = scp.SCPClient(transport)

        if action == 'get':
            print("\n{}scp {}:{} {}\n".format(shakedown.cli.helpers.fchr('>>'), host, remote_path, file_path))
            channel.get(remote_path, file_path)
        else:
            print("\n{}scp {} {}:{}\n".format(shakedown.cli.helpers.fchr('>>'), file_path, host, remote_path))
            channel.put(file_path, remote_path)

        print("{} bytes copied in {} seconds.".format(str(os.path.getsize(file_path)), str(round(time.time() - start, 2))))

        try_close(channel)
        try_close(transport)

        return True
    else:
        print("error: unable to authenticate {}@{} with key {}".format(username, host, key_path))
        return False