Python cStringIO.BytesIO() Examples

def makeHTTPRequest(url, query_params=None, content_coding_token="gzip", referer=None):
    """Makes an HTTP request and returns the JSON response. content_coding_token
       must be gzip or identity.
       If content_coding_token is identity, response does not need any transformation.
       If content_coding_token is gzip, the response special handling before converting to JSON."""

    response_dict = {}
    if query_params == None:
        query_params = {}
    if not "f" in query_params:
        query_params["f"] = "json"

    request = urllib2.Request(url)
    if ispy3:
        data = urllib.urlencode(query_params)
        binary_data = data.encode('utf-8')
        request.data = binary_data
    else:        
        request.add_data(urllib.urlencode(query_params))
    request.add_header("Accept-Encoding", content_coding_token)
    if referer:
        request.add_header("Referer", referer)
    response = urllib2.urlopen(request)
    if content_coding_token == "gzip":
        if response.info().get("Content-Encoding") == "gzip":
            if ispy3:
                # Encoding is complicated in python3
                buf = sio.BytesIO(response.read())
                response = gzip.GzipFile(fileobj=buf, mode='rb')
                reader = codecs.getreader("utf-8")
                response = reader(response)
            else:
                buf = sio.StringIO(response.read())
                response = gzip.GzipFile(fileobj=buf)
    response_dict = json.load(response)
    return response_dict 

def test_write_git_changelog(self):
        self.useFixture(fixtures.FakePopen(lambda _: {
            "stdout": BytesIO(_changelog_content.encode('utf-8'))
        }))

        git.write_git_changelog(git_dir=self.git_dir,
                                dest_dir=self.temp_path)

        with open(os.path.join(self.temp_path, "ChangeLog"), "r") as ch_fh:
            changelog_contents = ch_fh.read()
            self.assertIn("2013.2", changelog_contents)
            self.assertIn("0.5.17", changelog_contents)
            self.assertIn("------", changelog_contents)
            self.assertIn("Refactor hooks file", changelog_contents)
            self.assertIn(
                "Bug fix: create_stack() fails when waiting",
                changelog_contents)
            self.assertNotIn("Refactor hooks file.", changelog_contents)
            self.assertNotIn("182feb3", changelog_contents)
            self.assertNotIn("review/monty_taylor/27519", changelog_contents)
            self.assertNotIn("0.5.13", changelog_contents)
            self.assertNotIn("0.6.7", changelog_contents)
            self.assertNotIn("12", changelog_contents)
            self.assertNotIn("(evil)", changelog_contents)
            self.assertNotIn("ev()il", changelog_contents)
            self.assertNotIn("ev(il", changelog_contents)
            self.assertNotIn("ev)il", changelog_contents)
            self.assertNotIn("e(vi)l", changelog_contents)
            self.assertNotIn('Merge "', changelog_contents)
            self.assertNotIn('1_foo.1', changelog_contents) 

def _to_dataframe(self):
        """Read and return the dataset contents as a pandas DataFrame."""
        #TODO: figure out why passing in the opened stream directly gives invalid data
        data = self.read_as_binary()
        reader = BytesIO(data)
        return deserialize_dataframe(reader, self.data_type_id) 

def write(self, data, msg_type, **options):
        '''Serializes and pushes single data object to a wrapped stream.
        
        :Parameters:
         - `data` - data to be serialized
         - `msg_type` (one of the constants defined in :class:`.MessageType`) -
           type of the message
        :Options:
         - `single_char_strings` (`boolean`) - if ``True`` single char Python 
           strings are encoded as q strings instead of chars, 
           **Default**: ``False``
        
        :returns: if wraped stream is ``None`` serialized data, 
                  otherwise ``None`` 
        '''
        self._buffer = BytesIO()

        self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))

        # header and placeholder for message size
        self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))).encode(self._encoding))

        self._write(data)

        # update message size
        data_size = self._buffer.tell()
        self._buffer.seek(4)
        self._buffer.write(struct.pack('i', data_size))

        # write data to socket
        if self._stream:
            self._stream.sendall(self._buffer.getvalue())
        else:
            return self._buffer.getvalue() 

def test_reading_numpy_temporals():
    BINARY = OrderedDict()

    with open('tests/QExpressions3.out', 'rb') as f:
        while True:
            query = f.readline().strip()
            binary = f.readline().strip()

            if not binary:
                break

            BINARY[query] = binary

    print('Deserialization (numpy temporals)')
    for query, value in iter(NUMPY_TEMPORAL_EXPRESSIONS.items()):
        buffer_ = BytesIO()
        binary = binascii.unhexlify(BINARY[query])

        buffer_.write(b'\1\0\0\0')
        buffer_.write(struct.pack('i', len(binary) + 8))
        buffer_.write(binary)
        buffer_.seek(0)

        sys.stdout.write( '  %-75s' % query )
        try:
            buffer_.seek(0)
            stream_reader = qreader.QReader(buffer_)
            result = stream_reader.read(numpy_temporals = True).data
            assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result)
            print('.')
        except QException as e:
            assert isinstance(value, QException)
            assert e.args == value.args
            print('.') 

def test_reading_compressed():
    BINARY = OrderedDict()

    with open('tests/QCompressedExpressions3.out', 'rb') as f:
        while True:
            query = f.readline().strip()
            binary = f.readline().strip()

            if not binary:
                break

            BINARY[query] = binary

    print('Compressed deserialization')
    buffer_reader = qreader.QReader(None)
    for query, value in iter(COMPRESSED_EXPRESSIONS.items()):
        buffer_ = BytesIO()
        binary = binascii.unhexlify(BINARY[query])

        buffer_.write(b'\1\0\1\0')
        buffer_.write(struct.pack('i', len(binary) + 8))
        buffer_.write(binary)
        buffer_.seek(0)

        sys.stdout.write( '  %-75s' % query )
        try:
            result = buffer_reader.read(source = buffer_.getvalue()).data
            assert compare(value, result), 'deserialization failed: %s' % (query)

            header = buffer_reader.read_header(source = buffer_.getvalue())
            result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed)
            assert compare(value, result), 'deserialization failed: %s' % (query)

            stream_reader = qreader.QReader(buffer_)
            result = stream_reader.read().data
            assert compare(value, result), 'deserialization failed: %s' % (query)
            print('.')
        except QException as e:
            assert isinstance(value, QException)
            assert e.args == value.args
            print('.') 

def test_reading_pandas():
        print('Deserialization (pandas)')
        for query, value in iter(PANDAS_EXPRESSIONS.items()):
            buffer_ = BytesIO()
            binary = binascii.unhexlify(BINARY[query])

            buffer_.write(b'\1\0\0\0')
            buffer_.write(struct.pack('i', len(binary) + 8))
            buffer_.write(binary)
            buffer_.seek(0)

            sys.stdout.write('  %-75s' % query)
            try:
                buffer_.seek(0)
                stream_reader = PandasQReader(buffer_)
                result = stream_reader.read(pandas = True).data
                if isinstance(value, dict):
                    if 'index' in value:
                        meta = result.meta
                        result = result.reset_index()
                        result.meta = meta

                    if not 'compare_meta' in value or value['compare_meta']:
                        assert value['meta'].as_dict() == result.meta.as_dict(), 'deserialization failed qtype: %s, expected: %s actual: %s' % (query, value['meta'], result.meta)
                    assert compare(value['data'], result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value['data'], result)
                else:
                    assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result)
                print('.')
            except QException as e:
                assert isinstance(value, QException)
                assert e.message == value.message
                print('.') 

def test_write_git_changelog(self):
        self.useFixture(fixtures.FakePopen(lambda _: {
            "stdout": BytesIO(_changelog_content.encode('utf-8'))
        }))

        git.write_git_changelog(git_dir=self.git_dir,
                                dest_dir=self.temp_path)

        with open(os.path.join(self.temp_path, "ChangeLog"), "r") as ch_fh:
            changelog_contents = ch_fh.read()
            self.assertIn("2013.2", changelog_contents)
            self.assertIn("0.5.17", changelog_contents)
            self.assertIn("------", changelog_contents)
            self.assertIn("Refactor hooks file", changelog_contents)
            self.assertIn(
                "Bug fix: create_stack() fails when waiting",
                changelog_contents)
            self.assertNotIn("Refactor hooks file.", changelog_contents)
            self.assertNotIn("182feb3", changelog_contents)
            self.assertNotIn("review/monty_taylor/27519", changelog_contents)
            self.assertNotIn("0.5.13", changelog_contents)
            self.assertNotIn("0.6.7", changelog_contents)
            self.assertNotIn("12", changelog_contents)
            self.assertNotIn("(evil)", changelog_contents)
            self.assertNotIn("ev()il", changelog_contents)
            self.assertNotIn("ev(il", changelog_contents)
            self.assertNotIn("ev)il", changelog_contents)
            self.assertNotIn("e(vi)l", changelog_contents)
            self.assertNotIn('Merge "', changelog_contents)
            self.assertNotIn('1_foo.1', changelog_contents) 

def _update_from_dataframe(self, dataframe, data_type_id=None, name=None,
                              description=None):
        """
        Serialize the specified DataFrame and replace the existing dataset.

        Parameters
        ----------
        dataframe : pandas.DataFrame
            Data to serialize.
        data_type_id : str, optional
            Format to serialize to.
            If None, the existing format is preserved.
            Supported formats are:
                'PlainText'
                'GenericCSV'
                'GenericTSV'
                'GenericCSVNoHeader'
                'GenericTSVNoHeader'
            See the azureml.DataTypeIds class for constants.
        name : str, optional
            Name for the dataset.
            If None, the name of the existing dataset is used.
        description : str, optional
            Description for the dataset.
            If None, the name of the existing dataset is used.
        """
        _not_none('dataframe', dataframe)

        if data_type_id is None:
            data_type_id = self.data_type_id
        if name is None:
            name = self.name
        if description is None:
            description = self.description

        try:
            output = BytesIO()
            serialize_dataframe(output, data_type_id, dataframe)
            raw_data = output.getvalue()
        finally:
            output.close()

        self._upload_and_refresh(raw_data, data_type_id, name, description) 

def add_from_dataframe(self, dataframe, data_type_id, name, description):
        """
        Serialize the specified DataFrame and upload it as a new dataset.

        Parameters
        ----------
        dataframe : pandas.DataFrame
            Data to serialize.
        data_type_id : str
            Format to serialize to.
            Supported formats are:
                'PlainText'
                'GenericCSV'
                'GenericTSV'
                'GenericCSVNoHeader'
                'GenericTSVNoHeader'
            See the azureml.DataTypeIds class for constants.
        name : str
            Name for the new dataset.
        description : str
            Description for the new dataset.

        Returns
        -------
        SourceDataset
            Dataset that was just created.
            Use open(), read_as_binary(), read_as_text() or to_dataframe() on
            the dataset object to get its contents as a stream, bytes, str or
            pandas DataFrame.
        """
        _not_none('dataframe', dataframe)
        _not_none_or_empty('data_type_id', data_type_id)
        _not_none_or_empty('name', name)
        _not_none_or_empty('description', description)

        try:
            output = BytesIO()
            serialize_dataframe(output, data_type_id, dataframe)
            raw_data = output.getvalue()
        finally:
            output.close()

        return self._upload(raw_data, data_type_id, name, description) 

def test_reading():
    BINARY = OrderedDict()

    with open('tests/QExpressions3.out', 'rb') as f:
        while True:
            query = f.readline().strip()
            binary = f.readline().strip()

            if not binary:
                break

            BINARY[query] = binary

    buffer_reader = qreader.QReader(None)
    print('Deserialization')
    for query, value in iter(EXPRESSIONS.items()):
        buffer_ = BytesIO()
        binary = binascii.unhexlify(BINARY[query])

        buffer_.write(b'\1\0\0\0')
        buffer_.write(struct.pack('i', len(binary) + 8))
        buffer_.write(binary)
        buffer_.seek(0)

        sys.stdout.write( '  %-75s' % query )
        try:
            header = buffer_reader.read_header(source = buffer_.getvalue())
            result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed, raw = True)
            assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query)

            stream_reader = qreader.QReader(buffer_)
            result = stream_reader.read(raw = True).data
            assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query)

            result = buffer_reader.read(source = buffer_.getvalue()).data
            assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result)

            header = buffer_reader.read_header(source = buffer_.getvalue())
            result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed)
            assert compare(value, result), 'deserialization failed: %s' % (query)

            buffer_.seek(0)
            stream_reader = qreader.QReader(buffer_)
            result = stream_reader.read().data
            assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result)
            print('.')
        except QException as e:
            assert isinstance(value, QException)
            assert e.args == value.args
            print('.')