Python docker.DockerClient() Examples

def initiate_remote_docker_connection(self, docker_server_url):
        """
        Docker allows remote access to its API in a number of ways.
        This function takes in a parameter of a server url and attempts to establish a docker connection

        For example 'ssh://admin:pass@192.168.1.0'
        :param docker_server_url:
        :return: void
        """
        try:
            self.client = docker.DockerClient(base_url=docker_server_url)
            self.api_client = docker.APIClient(base_url=docker_server_url)
            LOG.debug("The DockerClient version is {}".format(self.client.version()['Version']))
            self.client.ping()
        except ConnectionError:
            LOG.error('Error connecting to docker')
            raise ValueError("Could not setup Docker connection, is docker running ?") 

def create_client():
    """
    Create a client to either a Docker instance.
    """
    kwargs = {
        "base_url": os.environ.get("DOCKER_HOST"),
        "timeout": 15,  # wait a bit, but give up before 30s Heroku request timeout
    }

    if os.environ.get("DOCKER_TLS_VERIFY"):
        kwargs["tls"] = TLSConfig(
            client_cert=(
                env_to_file("DOCKER_CLIENT_CERT"),
                env_to_file("DOCKER_CLIENT_KEY"),
            ),
            ca_cert=env_to_file("DOCKER_CA_CERT"),
            verify=True,
        )

    return docker.DockerClient(**kwargs) 

def __init__(self, docker_config: DockerHostConfig, mock_client=None) -> None:
        self.name = docker_config.name
        self.docker_config = docker_config
        if not docker_config.tls:
            tls = None
        else:
            tls = docker.tls.TLSConfig(client_cert=(docker_config.tls_cert, docker_config.tls_key), verify=docker_config.tls_ca)

        # Simplify testing
        if mock_client is not None:
            self.cli = mock_client
            return

        try:
            self.cli = docker.DockerClient(base_url=docker_config.address, version="auto", tls=tls)
        except docker.errors.DockerException as e:
            raise ZoeException("Cannot connect to Docker host {} at address {}: {}".format(docker_config.name, docker_config.address, str(e))) 

def create_docker_client():
    # In order to build and push to the minishift registry, it's required that
    # users have configured their shell to use the minishift docker daemon
    # instead of a local daemon:
    # https://docs.openshift.org/latest/minishift/using/docker-daemon.html
    if is_minishift():
        cert_path = os.environ.get('DOCKER_CERT_PATH')
        docker_host = os.environ.get('DOCKER_HOST')
        if docker_host is None or cert_path is None:
            raise Exception("Attempting to target minishift, but missing required \
                            env vars. Try running: \"eval $(minishift docker-env)\"")
        client_cert = os.path.join(cert_path, 'cert.pem')
        client_key = os.path.join(cert_path, 'key.pem')
        ca_cert = os.path.join(cert_path, 'ca.pem')
        tls = docker.tls.TLSConfig(
            ca_cert=ca_cert,
            client_cert=(client_cert, client_key),
            verify=True,
            assert_hostname=False
        )
        client = docker.DockerClient(tls=tls, base_url=docker_host, version='auto')
    else:
        client = docker.DockerClient(base_url='unix://var/run/docker.sock', version='auto')
    return client 

def __init__(self, master_host, slave_hosts,
                 local_mount_dir, docker_mount_dir):
        # see PyDoc for all_internal_hosts() for an explanation on the
        # difference between an internal and regular host
        self.internal_master = master_host
        self.internal_slaves = slave_hosts
        self._master = master_host + '-' + str(uuid.uuid4())
        self.slaves = [slave + '-' + str(uuid.uuid4())
                       for slave in slave_hosts]
        # the root path for all local mount points; to get a particular
        # container mount point call get_local_mount_dir()
        self.local_mount_dir = local_mount_dir
        self._mount_dir = docker_mount_dir

        kwargs = kwargs_from_env()
        if 'tls' in kwargs:
            kwargs['tls'].assert_hostname = False
        kwargs['timeout'] = 300
        self.client = DockerClient(**kwargs)
        self._user = 'root'
        self._network_name = 'presto-admin-test-' + str(uuid.uuid4())

        DockerCluster.__check_if_docker_exists() 

def create_docker_client(docker_host: str = '', check=True) -> docker.DockerClient:
    """
    Context manager for DockerClient creation

    :param docker_host: DOCKER_HOST arg for DockerClient
    :param check: check if docker is available
    :return: DockerClient instance
    """
    with _docker_host_lock:
        os.environ["DOCKER_HOST"] = docker_host  # The env var DOCKER_HOST is used to configure docker.from_env()
        client = docker.from_env()
    if check and not _is_docker_running(client):
        raise RuntimeError("Docker daemon is unavailable")
    try:
        yield client
    finally:
        client.close() 

def get_docker_client(base_url=None, retry_read=config.DOCKER_MAX_READ_RETRIES,
                      retry_status_forcelist=(500,)):
    client_key = (retry_read, retry_status_forcelist)
    if client_key not in _DOCKER_CLIENTS:
        client = docker.DockerClient(base_url=base_url or config.DOCKER_URL,
                                     timeout=config.DOCKER_TIMEOUT)
        retries = Retry(total=config.DOCKER_MAX_TOTAL_RETRIES,
                        connect=config.DOCKER_MAX_CONNECT_RETRIES,
                        read=retry_read,
                        method_whitelist=False,
                        status_forcelist=retry_status_forcelist,
                        backoff_factor=config.DOCKER_BACKOFF_FACTOR,
                        raise_on_status=False)
        http_adapter = HTTPAdapter(max_retries=retries)
        client.api.mount('http://', http_adapter)
        _DOCKER_CLIENTS[client_key] = client
    return _DOCKER_CLIENTS[client_key] 

def _pull_predefined_dockerimages(self):
        """
        If the package contains URLs to pre-build Docker images, we download them with this method.
        """
        dc = DockerClient()
        for url in self.remote_docker_image_urls.itervalues():
            # only pull if not present (speedup for development)
            if not FORCE_PULL:
                if len(dc.images.list(name=url)) > 0:
                    LOG.debug("Image %r present. Skipping pull." % url)
                    continue
            LOG.info("Pulling image: %r" % url)
            # this seems to fail with latest docker api version 2.0.2
            # dc.images.pull(url,
            #        insecure_registry=True)
            # using docker cli instead
            cmd = ["docker",
                   "pull",
                   url,
                   ]
            Popen(cmd).wait() 

def connect(self, params={}):
        client_cert = (
            helper.key_to_file(params.get('client_cert').get('secretKey')),
            helper.key_to_file(params.get('client_key').get('secretKey'))
        )
        ca_cert = helper.key_to_file(params.get('ca_cert').get('secretKey'))
        tls_config = docker.tls.TLSConfig(
            client_cert=client_cert,
            ca_cert=ca_cert
        )
        base_url = params.get('url')

        try:
            self.logger.info("Connect: Connecting to {}".format(base_url))
            self.docker_client = docker.DockerClient(
                base_url=base_url,
                tls=tls_config,
                version=params.get('api_version')
            )
        except docker.errors.DockerException:
            raise
        else:
            self.logger.info("Connect: Connected to {} successfully.".format(base_url)) 

def __init__(self):
        """ Connnects to the docker daemon"""
        # will be used as the tag on the docker image
        self.problem_name = sanitize_name(self.name)
        # use an explicit remote docker daemon per the configuration
        try:
            tls_config = docker.tls.TLSConfig(
                ca_cert=self.docker_ca_cert,
                client_cert=(self.docker_client_cert, self.docker_client_key),
                verify=True)

            self.client = docker.DockerClient(base_url=self.docker_host, tls=tls_config)
            self.api_client = docker.APIClient(base_url=self.docker_host, tls=tls_config)
            logger.debug("Connecting to docker daemon with config")

        # Docker options not set in configuration so use the environment to
        # configure (could be local or remote)
        except AttributeError:
            logger.debug("Connecting to docker daemon with env")
            self.client = docker.from_env()

        # throws an exception if the server returns an error: docker.errors.APIError
        self.client.ping() 

def _setResourceAllocation(allocation):
    client = docker.DockerClient(base_url="unix://var/run/docker.sock", version='auto')
    for container_name, resources in six.iteritems(allocation):
        out.info("Update chute {} set cpu_shares={}\n".format(
            container_name, resources['cpu_shares']))
        container = client.containers.get(container_name)
        container.update(cpu_shares=resources['cpu_shares'])

        # Using class id 1:1 for prioritized, 1:3 for best effort.
        # Prioritization is implemented in confd/qos.py.  Class-ID is
        # represented in hexadecimal.
        # Reference: https://www.kernel.org/doc/Documentation/cgroup-v1/net_cls.txt
        if resources.get('prioritize_traffic', False):
            classid = "0x10001"
        else:
            classid = "0x10003"

        container = ChuteContainer(container_name)
        try:
            container_id = container.getID()
            fname = "/sys/fs/cgroup/net_cls/docker/{}/net_cls.classid".format(container_id)
            with open(fname, "w") as output:
                output.write(classid)
        except Exception as error:
            out.warn("Error setting traffic class: {}\n".format(error)) 

def __init__(self, context, spec, build_status=None, docker_version='auto'):
        self.context = context
        self.spec = spec
        self.repo_name = context.repository.split('/')[-1]
        self.commit_hash = context.source['commit']['hash']
        self.build_status = build_status or BuildStatus(
            bitbucket,
            context.source['repository']['full_name'],
            self.commit_hash,
            'badwolf/test',
            url_for('log.build_log', sha=self.commit_hash, _external=True)
        )

        self.docker = DockerClient(
            base_url=current_app.config['DOCKER_HOST'],
            timeout=current_app.config['DOCKER_API_TIMEOUT'],
            version=docker_version,
        ) 

def getBridgeGateway():
    """
    Look up the gateway IP address for the docker bridge network.

    This is the docker0 IP address; it is the IP address of the host from the
    chute's perspective.
    """
    client = docker.DockerClient(base_url="unix://var/run/docker.sock", version='auto')

    network = client.networks.get("bridge")
    for config in network.attrs['IPAM']['Config']:
        if 'Gateway' in config:
            return config['Gateway']

    # Fall back to a default if we could not find it.  This address will work
    # in most places unless Docker changes to use a different address.
    out.warn('Could not find bridge gateway, using default')
    return '172.17.0.1' 

def _UpdateNode(self, leader, node_id, node_role, labels={}):
        """ 更新节点信息(Labels、Role等) """

        client = docker.DockerClient(base_url="tcp://{}:{}".format(leader, self.port), version="auto", timeout=self.timeout)

        node_spec = {
            'Availability': 'active',
            'Role': node_role,
            'Labels': labels
        }
        logger.info("Update node spec data is {} for node_id {})".format(node_spec, node_id))

        try:
            node = client.nodes.get(node_id)
            res  = node.update(node_spec)
        except docker.errors.APIError,e:
            logger.error(e, exc_info=True)
            return False 

def check_docker(self):

        logger.info(f"testing docker API on {DOCKER_SOCKET}…")
        if (
            not DOCKER_SOCKET.exists()
            or not DOCKER_SOCKET.is_socket()
            or not os.access(DOCKER_SOCKET, os.R_OK)
        ):
            logger.critical(f"\tsocket ({DOCKER_SOCKET}) not available.")
            sys.exit(1)
        self.docker = docker.DockerClient(
            base_url=f"unix://{DOCKER_SOCKET}", timeout=DOCKER_CLIENT_TIMEOUT
        )
        try:
            if len(self.docker.containers.list(all=False)) < 1:
                logger.warning("\tno running container, am I out-of-docker?")
        except Exception as exc:
            logger.critical("\tdocker API access failed: exiting.")
            logger.exception(exc)
            sys.exit(1)
        else:
            logger.info("\tdocker API access successful") 

def connect(self):
        if self.config.docker_tls:
            try:
                cert_paths = {
                    'cert_top_dir': '/etc/docker/certs.d/',
                    'clean_socket': self.socket.split('//')[1],
                }
                cert_paths['cert_dir'] = join(cert_paths['cert_top_dir'], cert_paths['clean_socket'])
                cert_paths['cert_files'] = {
                    'client_cert': join(cert_paths['cert_dir'], 'client.cert'),
                    'client_key': join(cert_paths['cert_dir'], 'client.key'),
                    'ca_crt': join(cert_paths['cert_dir'], 'ca.crt'),
                }

                if not isdir(cert_paths['cert_dir']):
                    self.logger.error('%s is not a valid cert folder', cert_paths['cert_dir'])
                    raise ValueError

                for cert_file in cert_paths['cert_files'].values():
                    if not isfile(cert_file):
                        self.logger.error('%s does not exist', cert_file)
                        raise ValueError

                tls_config = tls.TLSConfig(
                    ca_cert=cert_paths['cert_files']['ca_crt'],
                    verify=cert_paths['cert_files']['ca_crt'] if self.config.docker_tls_verify else False,
                    client_cert=(cert_paths['cert_files']['client_cert'], cert_paths['cert_files']['client_key']),
                )
                client = DockerClient(base_url=self.socket, tls=tls_config)
            except ValueError:
                self.logger.error('Invalid Docker TLS config for %s, reverting to unsecured', self.socket)
                client = DockerClient(base_url=self.socket)
        else:
            client = DockerClient(base_url=self.socket)

        return client 

def _make_docker_client(config: infra_validator_pb2.LocalDockerConfig):
  params = {}
  if config.client_timeout_seconds:
    params['timeout'] = config.client_timeout_seconds
  if config.client_base_url:
    params['base_url'] = config.client_base_url
  if config.client_api_version:
    params['version'] = config.client_api_version
  return docker.DockerClient(**params) 

def remove_current_docker_container(app, user_id, is_retry=False):
        configs = DBUtils.get_all_configs()
        container = DBUtils.get_current_containers(user_id=user_id)

        auto_containers = configs.get("docker_auto_connect_containers", "").split(",")

        if container is None:
            return False

        try:
            client = docker.DockerClient(base_url=configs.get("docker_api_url"))
            networks = client.networks.list(names=[str(user_id) + '-' + container.uuid])

            if len(networks) == 0:
                services = client.services.list(filters={'name': str(user_id) + '-' + container.uuid})
                for s in services:
                    s.remove()
            else:
                redis_util = RedisUtils(app)
                services = client.services.list(filters={'label': str(user_id) + '-' + container.uuid})
                for s in services:
                    s.remove()

                for n in networks:
                    for ac in auto_containers:
                        try:
                            n.disconnect(ac, force=True)
                        except:
                            pass
                    n.remove()
                    redis_util.add_available_network_range(n.attrs['Labels']['prefix'])
        except:
            traceback.print_exc()
            if not is_retry:
                DockerUtils.remove_current_docker_container(app, user_id, True)

        return True 

def extract_output(docker_client: DockerClient, image_id: str,
                   output_dir: Path) -> None:
    image = docker_client.images.get(image_id)
    container = docker_client.containers.create(image)

    # Get archive tar bytes from the container as a sequence of bytes
    package_tar_byte_gen: typing.Generator[bytes, None, None]
    package_tar_byte_gen, _ = container.get_archive("/output/",
                                                    chunk_size=None)

    # Concat all the chunks together
    package_tar_bytes: bytes
    package_tar_bytes = b"".join(package_tar_byte_gen)

    # Create a tarfile from the tar bytes
    tar_file_object = io.BytesIO(package_tar_bytes)
    package_tar = tarfile.open(fileobj=tar_file_object)

    # Extract the files from the tarfile to the disk
    for tar_deb_info in package_tar.getmembers():
        # Ignore directories
        if not tar_deb_info.isfile():
            continue

        # Directory that will contain the output files
        output_dir.mkdir(parents=True, exist_ok=True)

        # Filename (without outer directory)
        tar_deb_info.name = Path(tar_deb_info.name).name

        # Extract
        package_tar.extract(tar_deb_info, output_dir) 

def build_image(docker_client: DockerClient, dockerfile: Path) -> str:
    image_name = "pyqt5-stubs"

    # Using low-level API so that we can log as it occurs instead of only
    # after build has finished/failed
    resp = docker_client.api.build(
        path=str(dockerfile.parent),
        rm=True,
        tag=image_name)

    image_id: str = typing.cast(str, None)
    for chunk in json_stream(resp):
        if 'error' in chunk:
            message = f"Error while building Dockerfile for " \
                      f"{image_name}:\n{chunk['error']}"
            print(message)
            raise DockerBuildError(message)

        elif 'stream' in chunk:
            print(chunk['stream'].rstrip('\n'))
            # Taken from the high level API implementation of build
            match = re.search(r'(^Successfully built |sha256:)([0-9a-f]+)$',
                              chunk['stream'])
            if match:
                image_id = match.group(2)

    if not image_id:
        message = f"Unknown Error while building Dockerfile for " \
                  f"{image_name}. Build did not return an image ID"
        raise DockerBuildError(message)

    return image_id 

def _check_docker_image_exists(self, image_name):
        """
        Query the docker service and check if the given image exists
        :param image_name: name of the docker image
        :return:
        """
        return len(DockerClient().images.list(name=image_name)) > 0 

def _check_for_images(master_image_name, slave_image_name, tag='latest'):
        master_repotag = '%s:%s' % (master_image_name, tag)
        slave_repotag = '%s:%s' % (slave_image_name, tag)
        client = DockerClient(timeout=180)
        images = client.images.list()
        has_master_image = False
        has_slave_image = False
        for image in images:
            if master_repotag in image.tags:
                has_master_image = True
            if slave_repotag in image.tags:
                has_slave_image = True
        return has_master_image and has_slave_image 

def __init__(
            self, base_master_name, base_slave_name, base_tag, tag_decoration):
        super(TagBareImageProvider, self).__init__(tag_decoration)
        self.base_master_name = base_master_name
        self.base_slave_name = base_slave_name
        self.base_tag = base_tag
        self.client = DockerClient() 

def create_client():
    logger = get_logger('create_client')
    try:
        if use_environ():
            logger.info('using environment to create docker client')
            c = docker.from_env()
        else:
            c = docker.DockerClient(base_url='unix://var/run/docker.sock', version="auto")
        # XXX ?
        c.run = run(c)
        return c
    except DockerException as e:
        logger.exception('Unable to connect to a docker socket')
        raise pytest.UsageError("Could not connect to a running docker socket: %s" % str(e)) 

def docker_checker():
    client = docker.DockerClient(base_url='unix:///var/run/docker.sock')
    actions = {'health_status: healthy': add_worker, 'destroy': remove_worker}

    # creates the necessary connection to make the sql calls if the master is ready
    conn = connect_to_master()

    # introspect the compose project used by this citus cluster
    my_hostname = environ['HOSTNAME']
    this_container = client.containers.get(my_hostname)
    compose_project = this_container.labels['com.docker.compose.project']

    # we only care about worker container health/destroy events from this cluster
    print("found compose project: %s" % compose_project, file=stderr)
    filters = {'event': list(actions),
               'label': ["com.docker.compose.project=%s" % compose_project,
                         "com.citusdata.role=Worker"],
               'type': 'container'}


    # touch a file to signal we're healthy, then consume events
    print('listening for events...', file=stderr)
    open(HEALTHCHECK_FILE, 'a').close()
    for event in client.events(decode=True, filters=filters):
        worker_name = event['Actor']['Attributes']['name']
        status = event['status']

        status=actions[status](conn, worker_name)


# implemented to make Docker exit faster (it sends sigterm) 

def main(args):
    data  = {
        'count': 0,
        'payload': {}
    }

    docker_configure_username_password(args)

    try:
        docker_client = docker.DockerClient(base_url=DOCKER_BASE_URL)
        docker_registry = get_registry(args.get('repository_uri'))
        docker_login_response = docker_login(docker_client, args.get('docker_username'), args.get('docker_password'), docker_registry)
        
        if DOCKER_LOGIN_SUCCEEDED == docker_login_response.get('Status'): 
            docker_push_response = docker_push(docker_client, args['repository_uri'],args['repository_tag'])
            if docker_push_response:
                data['count'] = 1
                data['payload'].update({
                    'repository_uri': args['repository_uri'],
                    'repository_tag': args['repository_tag']
                })
            else:
                data['count'] = 0
    except Exception as e:
        print(e, file=sys.stderr)
    
    return data 

def main(args):
    data  = {
        'count': 0,
        'payload': {}
    }
    try:
        aws_session = get_aws_session(args['aws_cli_profile'], args['aws_region'])
        ecr_client = aws_session.client('ecr')
        docker_client = docker.DockerClient(base_url=DOCKER_BASE_URL)

        token = ecr_client.get_authorization_token()
        docker_username, docker_password, docker_registry = get_docker_username_password_registry(token)
        docker_login_response = docker_login(docker_client,docker_username, docker_password, docker_registry)
        
        if DOCKER_LOGIN_SUCCEEDED == docker_login_response.get('Status'): 
            docker_push_response = docker_push(docker_client, args['aws_ecr_repository_uri'],args['aws_ecr_repository_tag'])
            if docker_push_response:
                data['count'] = 1
                data['payload'].update({
                    'aws_region': args['aws_region'],
                    'aws_ecr_repository_uri': args['aws_ecr_repository_uri'],
                    'aws_ecr_repository_tag': args['aws_ecr_repository_tag']
                })
            else:
                data['count'] = 0
    except Exception as e:
        print(e, file=sys.stderr)
    
    return data 

def ecr_pull(args, data):
    try:
        aws_session = get_aws_session(args['aws_cli_profile'], args['aws_region'])
        ecr_client = aws_session.client('ecr')
        token = ecr_client.get_authorization_token()

        docker_client = docker.DockerClient(base_url=DOCKER_BASE_URL)
        docker_username, docker_password, docker_registry = get_docker_username_password_registry(token)
        docker_login_response = docker_login(docker_client, docker_username, docker_password, docker_registry)

        if DOCKER_LOGIN_SUCCEEDED == docker_login_response.get('Status'):
            count = 0
            for tag in args['aws_ecr_repository_tags']:
                repo_tag = args['aws_ecr_repository_uri'] + ":" + tag
                try:
                    docker_pull_response = docker_pull(docker_client, repo_tag)
                    out = 'Pulled {}'.format(docker_pull_response)
                    count += 1
                    data['payload']['aws_ecr_repository_tags'].append(tag)
                    print(out)
                except Exception as e:
                    print(e, file=sys.stderr)

            data['count'] = count
            data['payload']['aws_ecr_repository_uri'] = args['aws_ecr_repository_uri']
            data['payload']['aws_region'] = args['aws_region'] 
    except Exception as e:
        print(e, file=sys.stderr) 

def ecr_pull_all_repos(args, data):
    ecr_repos = args.get('ecr_repos')
    count = 0
    
    try:
        if ecr_repos.get('aws_regions'):
            for region in ecr_repos.get('aws_regions'):
                aws_session = get_aws_session(args['aws_cli_profile'], region)
                ecr_client = aws_session.client('ecr')
                token = ecr_client.get_authorization_token()
        
                docker_client = docker.DockerClient(base_url=DOCKER_BASE_URL)
                docker_username, docker_password, docker_registry = get_docker_username_password_registry(token)
                docker_login_response = docker_login(docker_client, docker_username, docker_password, docker_registry)
    
                if ecr_repos.get('repositories_by_region'):
                    for repo in ecr_repos.get('repositories_by_region').get(region):
                        if repo.get('repositoryUri'):
                            docker_pull_response = docker_pull(docker_client, repo.get('repositoryUri'))
                            out = 'Pulled {}'.format(docker_pull_response)
                            count += 1
                            print(out)

        data['count'] = count
    except Exception as e:
        print(e, file=sys.stderr) 

def gcr_pull(args, data):
    try:
        docker_client = docker.DockerClient(base_url=DOCKER_BASE_URL)
        docker_registry = get_registry(args.get('repositories')[0])
        docker_login_response = docker_login(docker_client, args.get('docker_username'), args.get('docker_password'), docker_registry)

        if DOCKER_LOGIN_SUCCEEDED == docker_login_response.get('Status'):
            count = 0
            if args.get('repository_tags'):
                # pull provided tags
                for tag in args.get('repository_tags'):
                    repo = args.get('repositories')[0] + ':' + tag
                    try:
                        docker_pull(docker_client, repo)
                        data['payload']['repository_tags'].append(tag)
                        count += 1
                    except Exception as e:
                       print(e, file=sys.stderr)
            else:
                # pull all tags
                try:
                    repo = args.get('repositories')[0]
                    docker_pull(docker_client, repo)
                    count += 1
                except Exception as e:
                    print(e, file=sys.stderr)
        else:
            print('login failed')

        data['count'] = count
        data['payload']['repositories'] = args.get('repositories')

    except Exception as e:
        print(e, file=sys.stderr)