Python requests_toolbelt.MultipartEncoder() Examples

def uploadPhoto(self, photo, caption=None, upload_id=None, is_sidecar=None):
        if upload_id is None:
            upload_id = str(int(time.time() * 1000))
        data = {'upload_id': upload_id,
                '_uuid': self.uuid,
                '_csrftoken': self.token,
                'image_compression': '{"lib_name":"jt","lib_version":"1.3.0","quality":"87"}',
                'photo': ('pending_media_%s.jpg' % upload_id, open(photo, 'rb'), 'application/octet-stream', {'Content-Transfer-Encoding': 'binary'})}
        if is_sidecar:
            data['is_sidecar'] = '1'
        m = MultipartEncoder(data, boundary=self.uuid)
        self.s.headers.update({'X-IG-Capabilities': '3Q4=',
                               'X-IG-Connection-Type': 'WIFI',
                               'Cookie2': '$Version=1',
                               'Accept-Language': 'en-US',
                               'Accept-Encoding': 'gzip, deflate',
                               'Content-type': m.content_type,
                               'Connection': 'close',
                               'User-Agent': self.USER_AGENT})
        response = self.s.post(self.API_URL + "upload/photo/", data=m.to_string())
        if response.status_code == 200:
            if self.configure(upload_id, photo, caption):
                self.expose()
        return False 

def upload(self, url, filepath, content_type, **kwargs):
        counter = 0
        while True:
            counter += 1
            try:
                abs_fp = os.path.realpath(os.path.expanduser(filepath))
                m = MultipartEncoder(fields={'file': (os.path.basename(filepath), open(abs_fp, 'rb'), content_type)})
                r = super(APISession, self).post(url, data=m, headers={'Content-Type': m.content_type}, timeout=self.timeout, **kwargs)
                r.raise_for_status()
                time.sleep(self.request_interval)
            except (HTTPError, ConnectionError, ReadTimeout) as e:
                if self.__recoverable(e, url, 'GET', counter):
                    continue
                else:
                    self.logger.debug(f'Unrecoverable error: {e}')
                    raise e
            return r 

def uploadImageFile(self, print_id, imageBuf):
		try:
			m = MultipartEncoder(fields=[('file',('snapshot.jpg', imageBuf))])
			r = requests.post(
				"%s/prints/%s/image" % (self.apiHost, print_id),
				data= m,
				headers= {'Content-Type': m.content_type},
				auth= self.hmacAuth
			)
			m = None #Free the memory?
			status_code = r.status_code
		except:
			status_code = 500

		if status_code == 201:
			data = r.json()
			return data

		else:
			return None 

def main():
    args = parse_args()

    with open(args.filename, 'rb') as file_:
        content_type = mimetypes.guess_type(args.filename)[0]

        fields = {
            'name': 'hello world',
            'lines': 'first line\r\n\r\nsecond line',
            'file': (args.filename, file_, content_type),
        }

        body = MultipartEncoder(fields=fields).to_string()

        if args.decode:
            print(body.decode('utf-8'))
        else:
            print(body) 

def test_case_insensitive_content_type():
    content_type_header = 'Content-Type'

    for header_key in (
        content_type_header,
        content_type_header.lower(),
        content_type_header.upper(),
        'cOnTeNt-tYPe',
    ):
        target = ValueTarget()

        encoder = MultipartEncoder(fields={'value': 'hello world'})

        parser = StreamingFormDataParser(
            headers={header_key: encoder.content_type}
        )
        parser.register('value', target)

        parser.data_received(encoder.to_string())

        assert target.value == b'hello world' 

def test_basic_multiple():
    first = ValueTarget()
    second = ValueTarget()
    third = ValueTarget()

    encoder = MultipartEncoder(
        fields={'first': 'foo', 'second': 'bar', 'third': 'baz'}
    )

    parser = StreamingFormDataParser(
        headers={'Content-Type': encoder.content_type}
    )

    parser.register('first', first)
    parser.register('second', second)
    parser.register('third', third)

    parser.data_received(encoder.to_string())

    assert first.value == b'foo'
    assert second.value == b'bar'
    assert third.value == b'baz' 

def test_chunked_single():
    expected_value = 'hello world'

    target = ValueTarget()

    encoder = MultipartEncoder(fields={'value': expected_value})

    body = encoder.to_string()

    parser = StreamingFormDataParser(
        headers={'Content-Type': encoder.content_type}
    )
    parser.register('value', target)

    index = body.index(b'world')

    parser.data_received(body[:index])
    parser.data_received(body[index:])

    assert target.value == expected_value.encode('utf-8') 

def upload_child_file(self, name, local_path, existing_node=None):
        logging.info("Uploading %s to %s", local_path, self.node["name"])
        mime_type = _get_mimetype(name)
        m = requests_toolbelt.MultipartEncoder([
            ("metadata", json.dumps({
                "name": name,
                "kind": "FILE",
                "parents": [self.node["id"]],
            })),
            ("content", (name, open(local_path, "rb"), mime_type))])
        if existing_node:
            """
            # TODO: this is under-documented and currently 500s on Amazon's side
            node = CloudNode(DriveSink.instance().request_content(
                "%%snodes/%s/content" % existing_node.node["id"],
                method="put", data=m, headers={"Content-Type": m.content_type}))
            """
            old_info = DriveSink.instance().request_metadata(
                "%%s/trash/%s" % existing_node.node["id"], method="put")
        ds = DriveSink.instance()
        resp = ds.request_content("%snodes", method="post", data=m,
            headers={"Content-Type": m.content_type})
        if resp != None:
            node = CloudNode(resp)
            self._children[name] = node 

def login(self, username, password):
        url = "https://leetcode.com/accounts/login"
        
        params_data = {
            'csrfmiddlewaretoken': self.csrftoken,
            'login': username,
            'password':password,
            'next': 'problems'
        }
        headers = {'User-Agent': user_agent, 'Connection': 'keep-alive', 'Referer': 'https://leetcode.com/accounts/login/',
        "origin": "https://leetcode.com"}
        m = MultipartEncoder(params_data)   

        headers['Content-Type'] = m.content_type
        self.session.post(url, headers = headers, data = m, timeout = 10, allow_redirects = False)
        self.is_login = self.session.cookies.get('LEETCODE_SESSION') != None
        return self.is_login 

def upload_file(self, model, filename, query_parameters=None, uri=None):
        if not uri:
            uri = model.uri()

        # urllib3 still uses RFC2231, not compliant with our server
        # https://github.com/shazow/urllib3/issues/303
        # https://github.com/shazow/urllib3/pull/856
        filename_ascii = bytes(filename, 'utf-8').decode('ascii', 'ignore')
        m = MultipartEncoder(fields={"files[]": (filename_ascii, open(filename, 'rb'))})
        response = self._session.post("{}{}".format(self._base_url(), uri),
                                      auth=CytomineAuth(
                                          self._public_key, self._private_key,
                                          self._base_url(), self._base_path),
                                      headers=self._headers(content_type=m.content_type),
                                      params=query_parameters,
                                      data=m)

        if response.status_code == requests.codes.ok:
            model = model.populate(response.json())  # [model.callback_identifier.lower()])
            self._logger.info("File uploaded successfully to {}".format(uri))
        else:
            model = False
            self._logger.error("Error during file uploading to {}".format(uri))

        return model 

def upload_record_without_photo(self, record):
        """ 不带自拍，上传跑步记录
        """
        m = MultipartEncoder(
                fields={
                    'userId': str(self.studentID),
                    'duration': str(record.duration),
                    'date': str(record.date),                        # 后端好像会根据日期来判断是否重复发包
                    'detail': json.dumps(record.detail),             # 路径似乎并不会用来查重
                    'misc': json.dumps({"agent": "Android v1.2+"}),
                    'step': str(record.step),
                }
            )
        # self.logger.debug(record.__dict__)
        # self.logger.debug(m.to_string()) # 注意！ m.to_string() 只能输出一次，第二次将会输出空字节，因此不要用这个方法来调试！
        # return
        respJson = self.post("record/{userId}".format(userId=self.studentID),
            data = m.to_string(), headers = {
                'Content-Type': m.content_type
            }, auth=self.auth)

        if not respJson["data"]["verified"]:
            raise PKURunnerNotVerifiedError("record is not verified, check your running params setting.") 

def make_api_request(method, files=None, **kwargs):
    from requests_toolbelt import MultipartEncoder
    if files is None:
        files = {}
    url = "http://" + str(host) + ":" + str(connection_port) + "/" + method

    kwargs = {k: v for k, v in kwargs.items() if v is not None}

    if len(files) > 0:
        fields = {}
        fields.update(kwargs)
        fields.update(files)
        m = MultipartEncoder(fields=fields)
        response = requests.post(url, data=m, headers={'Content-Type': m.content_type})
    else:
        response = requests.post(url, data=kwargs)
    if response.status_code != 200:
        return {
            'error': response.status_code,
            'message': response.text
        }
    else:
        return response.json() 

def test_smoke():
    encoder = MultipartEncoder(fields={'name': 'hello'})

    parser = StreamingFormDataParser(
        headers={'Content-Type': encoder.content_type}
    )

    parser.data_received(encoder.to_string()) 

def test_chunked_multiple():
    expected_first_value = 'foo' * 1000
    expected_second_value = 'bar' * 1000
    expected_third_value = 'baz' * 1000

    first = ValueTarget()
    second = ValueTarget()
    third = ValueTarget()

    encoder = MultipartEncoder(
        fields={
            'first': expected_first_value,
            'second': expected_second_value,
            'third': expected_third_value,
        }
    )

    body = encoder.to_string()

    parser = StreamingFormDataParser(
        headers={'Content-Type': encoder.content_type}
    )

    parser.register('first', first)
    parser.register('second', second)
    parser.register('third', third)

    chunks = []
    size = 100

    while len(body):
        chunks.append(body[:size])
        body = body[size:]

    for chunk in chunks:
        parser.data_received(chunk)

    assert first.value == expected_first_value.encode('utf-8')
    assert second.value == expected_second_value.encode('utf-8')
    assert third.value == expected_third_value.encode('utf-8') 

def imageUploadOnly(self, imageFile):
        if re.match(r'[a-zA-z]+://[^\s]*', imageFile):
            # Upload Image from URL
            resp = requests.Session().get(
                url=imageFile,
                headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0'},
                timeout=120,
                stream=True,
                verify=False
            )
            fileContent = resp.content
            mimeType = resp.headers['content-type']
            fileName = '%s.%s' % (uuid4().hex, re.sub(r'image/', '', mimeType))
            payload = MultipartEncoder(
                fields={'image': (fileName, fileContent, mimeType)})
        else:
            # Upload Image from Local File
            fileContent = codecs.open(imageFile, 'rb')
            mimeType = 'image/%s' % (imghdr.what(fileContent))
            fileName = '%s.%s' % (uuid4().hex, re.sub(r'image/', '', mimeType))
            payload = MultipartEncoder(
                fields={'image': (fileName, fileContent, mimeType)})
        url = 'https://cloud.smartisan.com/apps/note/index.php?r=image/upload'
        resp = self.session.post(
            url=url,
            headers={'Content-Type': payload.content_type},
            data=self.genChunks(payload),
            timeout=120,
            stream=True
        )
        self.checkResponse(resp)
        content = json.loads(resp.content)
        respData = content['data']
        return respData

    # Supported File Formats: jpeg, png ; Maximum File Size: 5MB 

def test_multiple_files():
    txt_filename = 'file.txt'
    png_filename = 'image-600x400.png'

    with open_dataset(txt_filename) as dataset_:
        expected_txt = dataset_.read()

    with open_dataset(png_filename) as dataset_:
        expected_png = dataset_.read()

    txt_target = ValueTarget()
    png_target = ValueTarget()

    with open_dataset(txt_filename) as txt_file, open_dataset(
        png_filename
    ) as png_file:
        encoder = MultipartEncoder(
            fields={
                txt_filename: (txt_filename, txt_file, 'application/plain'),
                png_filename: (png_filename, png_file, 'image/png'),
            }
        )

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )

        parser.register(txt_filename, txt_target)
        parser.register(png_filename, png_target)

        parser.data_received(encoder.to_string())

        assert txt_target.value == expected_txt
        assert png_target.value == expected_png 

def test_register_after_data_received():
    encoder = MultipartEncoder(fields={'name': 'hello'})

    parser = StreamingFormDataParser(
        headers={'Content-Type': encoder.content_type}
    )
    parser.data_received(encoder.to_string())

    with pytest.raises(ParseFailedException):
        parser.register('name', ValueTarget()) 

def test_multiple_targets():
    filename = 'image-600x400.png'

    with open_dataset(filename) as dataset_:
        expected_data = dataset_.read()

    value_target = ValueTarget()
    sha256_target = SHA256Target()

    with open_dataset(filename) as file_:
        encoder = MultipartEncoder(
            fields={filename: (filename, file_, 'image/png')}
        )

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )

        parser.register(filename, value_target)
        parser.register(filename, sha256_target)

        assert not value_target.value
        assert sha256_target.value == hashlib.sha256(b'').hexdigest()

        parser.data_received(encoder.to_string())

        assert value_target.value == expected_data
        assert sha256_target.value == hashlib.sha256(expected_data).hexdigest() 

def do_test(self, original_data, dataset_name, last_part):
        with BytesIO(original_data) as dataset_:
            if last_part:
                fields = {
                    'name': 'hello world',
                    'file': ('file.dat', dataset_, 'binary/octet-stream'),
                }
            else:
                fields = {
                    'file': ('file.dat', dataset_, 'binary/octet-stream'),
                    'name': 'hello world',
                }

            encoder = MultipartEncoder(fields=fields)
            content_type = encoder.content_type
            multipart_data = encoder.to_string()

        useful_numbers = get_useful_numbers()
        self.assertEqual(len(useful_numbers), 880)

        idx = 0
        for chunksize in useful_numbers:
            idx += 1

            self.subTest(
                idx,
                'DifferentChunks.' + dataset_name,
                chunksize,
                original_data,
                content_type,
                multipart_data,
                'file.dat',
            )

        self.assertEqual(idx, len(useful_numbers)) 

def do_test(self, data, dataset_name):
        useful_numbers = get_useful_numbers()

        idx = 0
        for file_size in chain([0], useful_numbers):
            idx += 1

            original_data = data[0:file_size]
            with BytesIO(data[0:file_size]) as dataset_:
                fields = {
                    'file': ('file.dat', dataset_, 'binary/octet-stream')
                }
                encoder = MultipartEncoder(fields=fields)
                content_type = encoder.content_type
                multipart_data = encoder.to_string()

            self.subTest(
                idx,
                'DifferentFileSizes.' + dataset_name,
                1024,
                original_data,
                content_type,
                multipart_data,
                'file.dat',
            )

        self.assertEqual(idx, len(useful_numbers) + 1) 

def do_test(self, data, dataset_name):
        useful_numbers = get_useful_numbers(short_list=True)
        self.assertEqual(len(useful_numbers), 140)

        idx = 0
        for file_size in chain([0], useful_numbers):

            original_data = data[0:file_size]
            with BytesIO(data[0:file_size]) as dataset_:
                fields = {
                    'file': ('file.dat', dataset_, 'binary/octet-stream')
                }
                encoder = MultipartEncoder(fields=fields)
                content_type = encoder.content_type
                multipart_data = encoder.to_string()

            for chunksize in useful_numbers:
                if chunksize > file_size:
                    continue
                idx += 1

                self.subTest(
                    idx,
                    'StressMatrixTest.' + dataset_name,
                    chunksize,
                    original_data,
                    content_type,
                    multipart_data,
                    'file.dat',
                )

        self.assertEqual(
            idx, len(useful_numbers) * (len(useful_numbers) + 1) / 2
        ) 

def create(
            cls, client, image_data, metadata=None, public=False, wait=True):
        """Create an image.

        If metadata is provided, a multipart form data request is formed to
        push metadata and image together in a single request. The metadata must
        be a tar achive.

        `wait` parameter is now ignored, as the image fingerprint cannot be
        reliably determined consistently until after the image is indexed.
        """

        if wait is False:  # pragma: no cover
            warnings.warn(
                'Image.create wait parameter ignored and will be removed in '
                '2.3', DeprecationWarning)

        headers = {}
        if public:
            headers['X-LXD-Public'] = '1'

        if metadata is not None:
            # Image uploaded as chunked/stream (metadata, rootfs)
            # multipart message.
            # Order of parts is important metadata should be passed first
            files = collections.OrderedDict(
                metadata=('metadata', metadata, 'application/octet-stream'),
                rootfs=('rootfs', image_data, 'application/octet-stream'))
            data = MultipartEncoder(files)
            headers.update({"Content-Type": data.content_type})
        else:
            data = image_data

        response = client.api.images.post(data=data, headers=headers)
        operation = client.operations.wait_for_operation(
            response.json()['operation'])
        return cls(client, fingerprint=operation.metadata['fingerprint']) 

def test_parameter_starts_with_crlf():
    target = ValueTarget()

    encoder = MultipartEncoder(fields={'value': '\r\nworld'})

    parser = StreamingFormDataParser(
        headers={'Content-Type': encoder.content_type}
    )
    parser.register('value', target)

    parser.data_received(encoder.to_string())

    assert target.value == b'\r\nworld' 

def add_source(ctx, username, id, features, no_validation, token=None, indent=None):
    """Create/add a tileset source

    tilesets add-source <username> <id> <path/to/source/data>
    """
    mapbox_api = utils._get_api()
    mapbox_token = utils._get_token(token)
    s = utils._get_session()
    url = (
        f"{mapbox_api}/tilesets/v1/sources/{username}/{id}?access_token={mapbox_token}"
    )

    with tempfile.TemporaryFile() as file:
        for feature in features:
            if not no_validation:
                utils.validate_geojson(feature)
            file.write((json.dumps(feature) + "\n").encode("utf-8"))

        file.seek(0)
        m = MultipartEncoder(fields={"file": ("file", file)})
        resp = s.post(
            url,
            data=m,
            headers={
                "Content-Disposition": "multipart/form-data",
                "Content-type": m.content_type,
            },
        )

    if resp.status_code == 200:
        click.echo(json.dumps(resp.json(), indent=indent))
    else:
        raise errors.TilesetsError(resp.text) 

def main():
    args = parse_args()

    with open_data(args) as fd:
        encoder = MultipartEncoder(
            fields={'file': ('file', fd, args.content_type)}
        )

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )
        parser.register('file', ValueTarget())

        parser.data_received(encoder.to_string()) 

def upload_image(self, upload_host, filename, id_storage, id_project=None, 
                     properties=None, sync=False, protocol=None):
        if not protocol:
            protocol = self._protocol
        upload_host, protocol = self._parse_url(upload_host, protocol)
        upload_host = "{}://{}".format(protocol, upload_host)

        query_parameters = {
            "idStorage": id_storage,
            "cytomine": "{}://{}".format(self._protocol, self._host),
            "sync": sync
        }

        if id_project:
            query_parameters["idProject"] = id_project

        if properties:
            query_parameters["keys"] = ','.join(list(properties.keys()))
            query_parameters["values"] = ','.join(list(properties.values()))

        # urllib3 still uses RFC2231, not compliant with our server
        # https://github.com/shazow/urllib3/issues/303
        # https://github.com/shazow/urllib3/pull/856
        filename_ascii = bytes(filename, 'utf-8').decode('ascii', 'ignore')
        m = MultipartEncoder(fields={"files[]": (filename_ascii, open(filename, 'rb'))})
        response = self._session.post("{}/upload".format(upload_host),
                                      auth=CytomineAuth(
                                          self._public_key, self._private_key,
                                          upload_host, ""),
                                      headers=self._headers(content_type=m.content_type),
                                      params=query_parameters,
                                      data=m)

        if response.status_code == requests.codes.ok:
            uf = self._process_upload_response(response.json()[0])
            self._logger.info("Image uploaded successfully to {}".format(upload_host))
            return uf
        else:
            self._logger.error("Error during image upload.")
            return False 

def __repr__(self):
        return '<MultipartEncoder: {!r}>'.format(self.fields) 

def __init__(self, encoder, callback=None):
        #: Instance of the :class:`MultipartEncoder` being monitored
        self.encoder = encoder

        #: Optionally function to call after a read
        self.callback = callback or IDENTITY

        #: Number of bytes already read from the :class:`MultipartEncoder`
        #: instance
        self.bytes_read = 0

        #: Avoid the same problem in bug #80
        self.len = self.encoder.len 

def upload_with_progress_bar(data, url, kwargs, label=None, token=None):
    """
    Uses multipart data to show progress of upload to the user.
    Requires request.files['data'].read() instead of request.data on the backend site.
    :param data: path to file to upload
    :param url: target url
    :param kwargs: additional args for the post request
    :param label: label of progress bar
    :param token: token to authorize the request
    :return: response from server
    """
    encoder = MultipartEncoder({'data': ('data', data, 'text/plain')})

    with tqdm(desc=label,
              total=encoder.len,
              disable=not label,
              dynamic_ncols=True,
              unit='B',
              unit_scale=True,
              unit_divisor=1024) as bar:
        multipart_monitor = MultipartEncoderMonitor(encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n))
        r = requests.post(url,
                          data=multipart_monitor,
                          headers={'Content-Type': multipart_monitor.content_type, 'X-Token': token},
                          params=kwargs)

    return r 

def from_fields(cls, fields, boundary=None, encoding='utf-8',
                    callback=None):
        encoder = MultipartEncoder(fields, boundary, encoding)
        return cls(encoder, callback)