Python bz2.compress() Examples

def test_queueMessages_handled_invalid_nodekey_with_instant_msg(self):
        worker = StatusWorkerService(sentinel.dbtasks)
        mock_processMessage = self.patch(worker, "_processMessage")
        contents = b"These are the contents of the file."
        encoded_content = encode_as_base64(bz2.compress(contents))
        message = self.make_message()
        message["files"] = [
            {
                "path": "sample.txt",
                "encoding": "uuencode",
                "compression": "bzip2",
                "content": encoded_content,
            }
        ]
        nodes_with_tokens = yield deferToDatabase(self.make_nodes_with_tokens)
        node, token = nodes_with_tokens[0]
        yield deferToDatabase(token.delete)
        yield worker.queueMessage(token.key, message)
        self.assertThat(mock_processMessage, MockNotCalled()) 

def _test_partial_input(self, mode):
        class MyStringIO(StringIO.StringIO):
            hit_eof = False
            def read(self, n):
                if self.hit_eof:
                    raise AssertionError("infinite loop detected in tarfile.open()")
                self.hit_eof = self.pos == self.len
                return StringIO.StringIO.read(self, n)
            def seek(self, *args):
                self.hit_eof = False
                return StringIO.StringIO.seek(self, *args)

        data = bz2.compress(tarfile.TarInfo("foo").tobuf())
        for x in range(len(data) + 1):
            try:
                tarfile.open(fileobj=MyStringIO(data[:x]), mode=mode)
            except tarfile.ReadError:
                pass # we have no interest in ReadErrors 

def _write_repodata(package_dir, repodata_dict):
    data = json.dumps(repodata_dict, indent=2, sort_keys=True)
    # strip trailing whitespace
    data = '\n'.join(line.rstrip() for line in data.splitlines())
    # make sure we have newline at the end
    if not data.endswith('\n'):
        data += '\n'

    with open(os.path.join(package_dir,
                           'repodata.json'), 'w') as fo:
        fo.write(data)

    # compress repodata.json into the bz2 format. some conda commands still
    # need it
    bz2_path = os.path.join(package_dir, 'repodata.json.bz2')
    with open(bz2_path, 'wb') as fo:
        fo.write(bz2.compress(data.encode('utf-8'))) 

def test_status_with_file_bad_compression_fails(self):
        node = factory.make_Node(
            interface=True, status=NODE_STATUS.COMMISSIONING
        )
        contents = b"These are the contents of the file."
        encoded_content = encode_as_base64(bz2.compress(contents))
        payload = {
            "event_type": "finish",
            "result": "FAILURE",
            "origin": "curtin",
            "name": "commissioning",
            "description": "Commissioning",
            "timestamp": datetime.utcnow(),
            "files": [
                {
                    "path": "sample.txt",
                    "encoding": "base64",
                    "compression": "jpeg",
                    "content": encoded_content,
                }
            ],
        }
        with ExpectedException(ValueError):
            self.processMessage(node, payload) 

def _test_partial_input(self, mode):
        class MyStringIO(StringIO.StringIO):
            hit_eof = False
            def read(self, n):
                if self.hit_eof:
                    raise AssertionError("infinite loop detected in tarfile.open()")
                self.hit_eof = self.pos == self.len
                return StringIO.StringIO.read(self, n)
            def seek(self, *args):
                self.hit_eof = False
                return StringIO.StringIO.seek(self, *args)

        data = bz2.compress(tarfile.TarInfo("foo").tobuf())
        for x in range(len(data) + 1):
            try:
                tarfile.open(fileobj=MyStringIO(data[:x]), mode=mode)
            except tarfile.ReadError:
                pass # we have no interest in ReadErrors 

def _test_partial_input(self, mode):
        class MyStringIO(StringIO.StringIO):
            hit_eof = False
            def read(self, n):
                if self.hit_eof:
                    raise AssertionError("infinite loop detected in tarfile.open()")
                self.hit_eof = self.pos == self.len
                return StringIO.StringIO.read(self, n)
            def seek(self, *args):
                self.hit_eof = False
                return StringIO.StringIO.seek(self, *args)

        data = bz2.compress(tarfile.TarInfo("foo").tobuf())
        for x in range(len(data) + 1):
            try:
                tarfile.open(fileobj=MyStringIO(data[:x]), mode=mode)
            except tarfile.ReadError:
                pass # we have no interest in ReadErrors 

def load(cls, branch, kv_store, msg_cls, hash):
        #  Update the branch's config store
        blob = kv_store[hash]
        if cls.compress:
            blob = decompress(blob)
        data = loads(blob)

        config_hash = data['config']
        config_data = cls.load_config(kv_store, msg_cls, config_hash)

        children_list = data['children']
        assembled_children = {}
        node = branch._node
        for field_name, meta in children_fields(msg_cls).iteritems():
            child_msg_cls = tmp_cls_loader(meta.module, meta.type)
            children = []
            for child_hash in children_list[field_name]:
                child_node = node._mknode(child_msg_cls)
                child_node.load_latest(child_hash)
                child_rev = child_node.latest
                children.append(child_rev)
            assembled_children[field_name] = children
        rev = cls(branch, config_data, assembled_children)
        return rev 

def _test_partial_input(self, mode):
        class MyStringIO(StringIO.StringIO):
            hit_eof = False
            def read(self, n):
                if self.hit_eof:
                    raise AssertionError("infinite loop detected in tarfile.open()")
                self.hit_eof = self.pos == self.len
                return StringIO.StringIO.read(self, n)
            def seek(self, *args):
                self.hit_eof = False
                return StringIO.StringIO.seek(self, *args)

        data = bz2.compress(tarfile.TarInfo("foo").tobuf())
        for x in range(len(data) + 1):
            try:
                tarfile.open(fileobj=MyStringIO(data[:x]), mode=mode)
            except tarfile.ReadError:
                pass # we have no interest in ReadErrors 

def test_status_with_file_bad_encoder_fails(self):
        node = factory.make_Node(
            interface=True, status=NODE_STATUS.COMMISSIONING
        )
        contents = b"These are the contents of the file."
        encoded_content = encode_as_base64(bz2.compress(contents))
        payload = {
            "event_type": "finish",
            "result": "FAILURE",
            "origin": "curtin",
            "name": "commissioning",
            "description": "Commissioning",
            "timestamp": datetime.utcnow(),
            "files": [
                {
                    "path": "sample.txt",
                    "encoding": "uuencode",
                    "compression": "bzip2",
                    "content": encoded_content,
                }
            ],
        }
        with ExpectedException(ValueError):
            self.processMessage(node, payload) 

def _test_partial_input(self, mode):
        class MyBytesIO(io.BytesIO):
            hit_eof = False
            def read(self, n):
                if self.hit_eof:
                    raise AssertionError("infinite loop detected in "
                                         "tarfile.open()")
                self.hit_eof = self.tell() == len(self.getvalue())
                return super(MyBytesIO, self).read(n)
            def seek(self, *args):
                self.hit_eof = False
                return super(MyBytesIO, self).seek(*args)

        data = bz2.compress(tarfile.TarInfo("foo").tobuf())
        for x in range(len(data) + 1):
            try:
                tarfile.open(fileobj=MyBytesIO(data[:x]), mode=mode)
            except tarfile.ReadError:
                pass # we have no interest in ReadErrors 

def test_queueMessages_processes_files_message_instantly(self):
        worker = StatusWorkerService(sentinel.dbtasks)
        mock_processMessage = self.patch(worker, "_processMessage")
        contents = b"These are the contents of the file."
        encoded_content = encode_as_base64(bz2.compress(contents))
        message = self.make_message()
        message["files"] = [
            {
                "path": "sample.txt",
                "encoding": "uuencode",
                "compression": "bzip2",
                "content": encoded_content,
            }
        ]
        nodes_with_tokens = yield deferToDatabase(self.make_nodes_with_tokens)
        node, token = nodes_with_tokens[0]
        yield worker.queueMessage(token.key, message)
        self.assertThat(mock_processMessage, MockCalledOnceWith(node, message)) 

def _test_partial_input(self, mode):
        class MyBytesIO(io.BytesIO):
            hit_eof = False
            def read(self, n):
                if self.hit_eof:
                    raise AssertionError("infinite loop detected in "
                                         "tarfile.open()")
                self.hit_eof = self.tell() == len(self.getvalue())
                return super(MyBytesIO, self).read(n)
            def seek(self, *args):
                self.hit_eof = False
                return super(MyBytesIO, self).seek(*args)

        data = bz2.compress(tarfile.TarInfo("foo").tobuf())
        for x in range(len(data) + 1):
            try:
                tarfile.open(fileobj=MyBytesIO(data[:x]), mode=mode)
            except tarfile.ReadError:
                pass # we have no interest in ReadErrors 

def _test_partial_input(self, mode):
        class MyStringIO(StringIO.StringIO):
            hit_eof = False
            def read(self, n):
                if self.hit_eof:
                    raise AssertionError("infinite loop detected in tarfile.open()")
                self.hit_eof = self.pos == self.len
                return StringIO.StringIO.read(self, n)
            def seek(self, *args):
                self.hit_eof = False
                return StringIO.StringIO.seek(self, *args)

        data = bz2.compress(tarfile.TarInfo("foo").tobuf())
        for x in range(len(data) + 1):
            try:
                tarfile.open(fileobj=MyStringIO(data[:x]), mode=mode)
            except tarfile.ReadError:
                pass # we have no interest in ReadErrors 

def __init__(self, path="./libsimilarity/libsimilarity.so"):
        super(SIMILARITYNative, self).__init__(True)

        self._u = cdll.LoadLibrary( path )

        self._u.compress.restype = c_uint
        self._u.ncd.restype = c_int
        self._u.ncs.restype = c_int
        self._u.cmid.restype = c_int
        self._u.entropy.restype = c_double
        self._u.levenshtein.restype = c_uint

        self._u.kolmogorov.restype = c_uint
        self._u.bennett.restype = c_double
        self._u.RDTSC.restype = c_double

        self.__libsim_t = LIBSIMILARITY_T()

        self.set_compress_type( ZLIB_COMPRESS ) 

def test_deserialize_dataframe_bz2(self):
        """Test deserializing a compressed dataframe."""
        content = bz2.compress(
            b'\n'.join(
                [
                    b'a,b,c',
                    b'1,2,3',
                    b'4,5,6'
                ]
            )
        )

        result = reports.deserialize_dataframe(content)
        pd.util.testing.assert_frame_equal(
            result,
            pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=['a', 'b', 'c'])
        ) 

def _ncd(self, s1, s2, s1size=0, s2size=0):
        if s1size == 0:
            s1size = self.compress(s1)

        if s2size == 0:
            s2size = self.compress(s2)

        s3size = self.compress(s1+s2)

        smax = max(s1size, s2size)
        smin = min(s1size, s2size)

        res = (abs(s3size - smin)) / float(smax)
        if res > 1.0:
            res = 1.0

        return res, s1size, s2size, 0 

def test_get(self, zk_mock):
        """Test fetching a report.
        """
        content = '\n'.join([
            'a,b,c',
            '1,2,3',
            '4,5,6'
        ])
        zk_mock.get.return_value = (bz2.compress(content.encode()), None)

        result = self.report.get('foo')

        zk_mock.get.assert_called_with('/reports/foo')
        pd.util.testing.assert_frame_equal(
            result,
            pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=['a', 'b', 'c'])
        ) 

def binaryToAttributes(cls, binary):
        """
        Turn a bytestring, or None, into SimpleDB attributes.
        """
        if binary is None: return {u'numChunks': 0}
        assert isinstance(binary, bytes)
        assert len(binary) <= cls.maxBinarySize()
        # The use of compression is just an optimization. We can't include it in the maxValueSize
        # computation because the compression ratio depends on the input.
        compressed = bz2.compress(binary)
        if len(compressed) > len(binary):
            compressed = b'U' + binary
        else:
            compressed = b'C' + compressed
        encoded = base64.b64encode(compressed)
        assert len(encoded) <= cls._maxEncodedSize()
        n = cls.maxValueSize
        chunks = (encoded[i:i + n] for i in range(0, len(encoded), n))
        attributes = {cls._chunkName(i): chunk for i, chunk in enumerate(chunks)}
        attributes.update({u'numChunks': len(attributes)})
        return attributes 

def _test_partial_input(self, mode):
        class MyStringIO(StringIO.StringIO):
            hit_eof = False
            def read(self, n):
                if self.hit_eof:
                    raise AssertionError("infinite loop detected in tarfile.open()")
                self.hit_eof = self.pos == self.len
                return StringIO.StringIO.read(self, n)
            def seek(self, *args):
                self.hit_eof = False
                return StringIO.StringIO.seek(self, *args)

        data = bz2.compress(tarfile.TarInfo("foo").tobuf())
        for x in range(len(data) + 1):
            try:
                tarfile.open(fileobj=MyStringIO(data[:x]), mode=mode)
            except tarfile.ReadError:
                pass # we have no interest in ReadErrors 

def _test_partial_input(self, mode):
        class MyBytesIO(io.BytesIO):
            hit_eof = False
            def read(self, n):
                if self.hit_eof:
                    raise AssertionError("infinite loop detected in "
                                         "tarfile.open()")
                self.hit_eof = self.tell() == len(self.getvalue())
                return super(MyBytesIO, self).read(n)
            def seek(self, *args):
                self.hit_eof = False
                return super(MyBytesIO, self).seek(*args)

        data = bz2.compress(tarfile.TarInfo("foo").tobuf())
        for x in range(len(data) + 1):
            try:
                tarfile.open(fileobj=MyBytesIO(data[:x]), mode=mode)
            except tarfile.ReadError:
                pass # we have no interest in ReadErrors 

def testCompress(self):
        # "Test compress() function"
        data = bz2.compress(self.TEXT)
        self.assertEqual(self.decompress(data), self.TEXT) 

def compress_directory(files_to_compress: List[pathlib.Path], output_name: str) -> None:
    """
    Compress a directory using all supported compression methods
    :param files_to_compress: Files to compress into a single archive
    :param output_name: Name to give compressed archive
    """
    temp_dir = mtgjson4.COMPILED_OUTPUT_DIR.joinpath(output_name)

    # Copy files to temporary folder
    LOGGER.info(f"Creating temporary directory for {output_name}")
    temp_dir.mkdir(parents=True, exist_ok=True)
    for file in files_to_compress:
        shutil.copy(str(file), str(temp_dir))

    # Compress the archives
    LOGGER.info(f"Compressing {temp_dir.name}")
    with multiprocessing.Pool(multiprocessing.cpu_count()) as pool:
        # Compression methods
        pool.apply_async(shutil.make_archive, (temp_dir, "bztar", str(temp_dir)))
        pool.apply_async(shutil.make_archive, (temp_dir, "gztar", str(temp_dir)))
        pool.apply_async(shutil.make_archive, (temp_dir, "xztar", str(temp_dir)))
        pool.apply_async(shutil.make_archive, (temp_dir, "zip", str(temp_dir)))

        # Wait for compressions to finish
        pool.close()
        pool.join()

    # Delete the temporary folder
    LOGGER.info(f"Removing temporary directory for {output_name}")
    shutil.rmtree(temp_dir, ignore_errors=True) 

def testCompress(self):
        # "Test BZ2Compressor.compress()/flush()"
        bz2c = BZ2Compressor()
        self.assertRaises(TypeError, bz2c.compress)
        data = bz2c.compress(self.TEXT)
        data += bz2c.flush()
        self.assertEqual(self.decompress(data), self.TEXT) 

def compress(self, data):
        if self is CompressionAlgorithm.Uncompressed:
            return data

        if self is CompressionAlgorithm.ZIP:
            return zlib.compress(data)[2:-4]

        if self is CompressionAlgorithm.ZLIB:
            return zlib.compress(data)

        if self is CompressionAlgorithm.BZ2:
            return bz2.compress(data)

        raise NotImplementedError(self) 

def testCompressChunks10(self):
        # "Test BZ2Compressor.compress()/flush() with chunks of 10 bytes"
        bz2c = BZ2Compressor()
        n = 0
        data = ''
        while 1:
            str = self.TEXT[n*10:(n+1)*10]
            if not str:
                break
            data += bz2c.compress(str)
            n += 1
        data += bz2c.flush()
        self.assertEqual(self.decompress(data), self.TEXT) 

def compress_file(file_path: pathlib.Path) -> None:
    """
    Compress a single file using all supported compression methods
    :param file_path: Path of file to compress
    """
    LOGGER.info(f"Compressing {file_path.name}")
    __generic_compressor(file_path, ".bz2", bz2.compress)
    __generic_compressor(file_path, ".gz", gzip.compress)
    __generic_compressor(file_path, ".xz", lzma.compress)

    # Zip files are done a bit differently
    with zipfile.ZipFile(str(file_path) + ".zip", "w") as zip_out:
        zip_out.write(file_path, file_path.name, zipfile.ZIP_DEFLATED) 

def load_config(cls, kv_store, msg_cls, config_hash):
        blob = kv_store[config_hash]
        if cls.compress:
            blob = decompress(blob)

        # TODO use a loader later on
        data = msg_cls()
        data.ParseFromString(blob)
        return data 

def __generic_compressor(
    input_file_path: pathlib.Path,
    file_ending: str,
    compress_function: Callable[[Any], Any],
) -> None:
    """
    Compress a single file
    :param input_file_path: File to compress
    :param file_ending: Ending to add onto archive
    :param compress_function: Function that compresses
    """
    output_file_path = pathlib.Path(str(input_file_path) + file_ending)

    with input_file_path.open("rb") as f_in, output_file_path.open("wb") as f_out:
        f_out.write(compress_function(f_in.read())) 

def single_model_predict(loader, model, modelname):
    MAX_BOND_COUNT = 406
    out_str = "id,scalar_coupling_constant\n"
    dev = "cuda"
    #dev = "cpu"
    model = model.to(dev)
    model.eval()
    with torch.no_grad():
        for arr in tqdm(loader):
            x_idx, x_atom, x_atom_pos, x_bond, x_bond_dist, x_triplet, x_triplet_angle, x_quad, x_quad_angle, y = arr
            x_atom, x_atom_pos, x_bond, x_bond_dist, x_triplet, x_triplet_angle, x_quad, x_quad_angle, y = \
                x_atom.to(dev), x_atom_pos.to(dev), x_bond.to(dev), x_bond_dist.to(dev), \
                x_triplet.to(dev), x_triplet_angle.to(dev), x_quad.to(dev), x_quad_angle.to(dev), y.to(dev)

            x_bond, x_bond_dist, y = x_bond[:, :MAX_BOND_COUNT], x_bond_dist[:, :MAX_BOND_COUNT], y[:,:MAX_BOND_COUNT]
            y_pred, _ = model(x_atom, x_atom_pos, x_bond, x_bond_dist, x_triplet, x_triplet_angle, x_quad, x_quad_angle)
            y_pred_pad = torch.cat([torch.zeros(y_pred.shape[0], 1, y_pred.shape[2], device=y_pred.device), y_pred], dim=1)
            y_pred_scaled = y_pred_pad.gather(1,x_bond[:,:,1][:,None,:])[:,0,:] * y[:,:,2] + y[:,:,1]

            y_selected = y_pred_scaled.masked_select((x_bond[:,:,0] > 0) & (y[:,:,3] > 0)).cpu().numpy()
            ids_selected = y[:,:,0].masked_select((x_bond[:,:,0] > 0) & (y[:,:,3] > 0))

            if dev=='cuda':
                ids_selected = ids_selected.cpu()
            ids_selected = ids_selected.numpy()

            for id_, pred in zip(ids_selected, y_selected):
                out_str += "{0:d},{1:f}\n".format(int(id_), pred)
    with open(os.path.join(root,settings['SUBMISSION_DIR'],modelname+'.csv.bz2'), "wb") as f:
        f.write(bz2.compress(out_str.encode('utf-8')))
    return 

def _export_python_sample_ast():
    # Get what to export
    if sys.version_info > (3, ):
        filenames = filename1, filename3
    else:
        filenames = filename2,
    # Write
    for filename in filenames:
        filename_bz2 = filename[:-2] + 'bz2'
        code = open(filename, 'rb').read().decode()
        root = commonast.parse(code)
        ast_bin = bz2.compress(root.tojson(indent=None).encode())
        with open(filename_bz2, 'wb') as f:
            n = f.write(ast_bin) or 'some'
        print('wrote %s bytes to %s' % (n, filename_bz2))