Python zipfile.ZIP_DEFLATED Examples

def ZipDir(sourceDirectory, outputFilePath, outputHandler):
		currentDir = os.getcwd()
		try:
			os.chdir(sourceDirectory)
			#relroot = os.path.abspath(os.path.join(sourceDirectory, os.pardir))
			relroot = os.path.abspath(os.path.join(sourceDirectory))
			#with zipfile.ZipFile(outputFilePath, "w", zipfile.ZIP_DEFLATED) as zip:
			with zipfile.ZipFile(outputFilePath, "w") as zip:
				for root, dirs, files in os.walk(sourceDirectory):
					# add directory (needed for empty dirs)
					# this is commented out because Tomcat 8 will reject WAR files with "./" in them.
					#zip.write(root, os.path.relpath(root, relroot))
					for file in files:
						filename = os.path.join(root, file)
						if os.path.isfile(filename): # regular files only
							arcname = os.path.join(os.path.relpath(root, relroot), file)
							zip.write(filename, arcname)
			return True
		except Exception as e:
			outputHandler.outputMessage('Error creating zip file "%s" from directory "%s" - %s' % (outputFilePath, sourceDirectory, e))
			return False
		os.chdir(currentDir) 

def pickle_load(path, compression=False):
    """Unpickle a possible compressed pickle.

    Parameters
    ----------
    path: str
        path to the output file
    compression: bool
        if true assumes that pickle was compressed when created and attempts decompression.

    Returns
    -------
    obj: object
        the unpickled object
    """

    if compression:
        with zipfile.ZipFile(path, "r", compression=zipfile.ZIP_DEFLATED) as myzip:
            with myzip.open("data") as f:
                return pickle.load(f)
    else:
        with open(path, "rb") as f:
            return pickle.load(f) 

def get(self, *, tid: objectid.ObjectId):
    tdoc, tsdocs = await contest.get_and_list_status(self.domain_id, document.TYPE_CONTEST, tid)
    rnames = {}
    for tsdoc in tsdocs:
      for pdetail in tsdoc.get('detail', []):
        rnames[pdetail['rid']] = 'U{}_P{}_R{}'.format(tsdoc['uid'], pdetail['pid'], pdetail['rid'])
    output_buffer = io.BytesIO()
    zip_file = zipfile.ZipFile(output_buffer, 'a', zipfile.ZIP_DEFLATED)
    rdocs = record.get_multi(get_hidden=True, _id={'$in': list(rnames.keys())})
    async for rdoc in rdocs:
      zip_file.writestr(rnames[rdoc['_id']] + '.' + rdoc['lang'], rdoc['code'])
    # mark all files as created in Windows :p
    for zfile in zip_file.filelist:
      zfile.create_system = 0
    zip_file.close()

    await self.binary(output_buffer.getvalue(), 'application/zip',
                      file_name='{}.zip'.format(tdoc['title'])) 

def pickle_load(path, compression=False):
    """Unpickle a possible compressed pickle.

    Parameters
    ----------
    path: str
        path to the output file
    compression: bool
        if true assumes that pickle was compressed when created and attempts decompression.

    Returns
    -------
    obj: object
        the unpickled object
    """

    if compression:
        with zipfile.ZipFile(path, "r", compression=zipfile.ZIP_DEFLATED) as myzip:
            with myzip.open("data") as f:
                return pickle.load(f)
    else:
        with open(path, "rb") as f:
            return pickle.load(f) 

def pickle_load(path, compression=False):
    """Unpickle a possible compressed pickle.

    Parameters
    ----------
    path: str
        path to the output file
    compression: bool
        if true assumes that pickle was compressed when created and attempts decompression.

    Returns
    -------
    obj: object
        the unpickled object
    """

    if compression:
        with zipfile.ZipFile(path, "r", compression=zipfile.ZIP_DEFLATED) as myzip:
            with myzip.open("data") as f:
                return pickle.load(f)
    else:
        with open(path, "rb") as f:
            return pickle.load(f) 

def export_debug(self, output_path):
        """
        this method is used to generate a debug map for NEO debugger
        """
        file = open(output_path, 'rb')
        file_hash = hashlib.md5(file.read()).hexdigest()
        file.close()

        avm_name = os.path.splitext(os.path.basename(output_path))[0]

        debug_info = self.generate_avmdbgnfo(avm_name, file_hash)
        debug_json_filename = os.path.basename(output_path.replace('.avm', '.debug.json'))
        avmdbgnfo_filename = output_path.replace('.avm', '.avmdbgnfo')

        with zipfile.ZipFile(avmdbgnfo_filename, 'w', zipfile.ZIP_DEFLATED) as avmdbgnfo:
            avmdbgnfo.writestr(debug_json_filename, debug_info) 

def create_zip_file(files_for_export, filekey):
    """
    Takes a list of dictionaries, each with a file object and a name, zips up all the files with those names and returns a zip file buffer.
    """

    buffer = BytesIO()
    with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zip:
        for f in files_for_export:
            f[filekey].seek(0)
            zip.writestr(f["name"], f[filekey].read())

    zip.close()
    buffer.flush()
    zip_stream = buffer.getvalue()
    buffer.close()
    return zip_stream 

def write_json_file(self):
        # get export, project and json path
        export_path = bpy.path.abspath(self.scene.coa_export_path)
        json_path = os.path.join(export_path, self.project_name + "_data.json")
        zip_path = os.path.join(export_path, self.project_name + "_data.zip")

        # write json file
        if self.reduce_size:
            json_file = json.dumps(self.json_data, separators=(',', ':'))
        else:
            json_file = json.dumps(self.json_data, indent="  ", sort_keys=False)

        text_file = open(json_path, "w")
        text_file.write(json_file)
        text_file.close()

        zip_file = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
        zip_file.write(json_path, os.path.basename(json_path))
        zip_file.close() 

def _createArchiveFile(archive_path, abs_base_dir, abs_paths):

    with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as zipf:
        for file in abs_paths:
            if not os.path.exists(file):
                zipf.close()
                if os.path.isfile(archive_path):
                    os.remove(archive_path)

                raise BenchExecException(
                    "Missing file '{0}', cannot run benchmark without it.".format(
                        os.path.normpath(file)
                    )
                )

            if os.path.isdir(file):
                _zipdir(file, zipf, abs_base_dir)
            else:
                zipf.write(file, os.path.relpath(file, abs_base_dir)) 

def _zip_files(dir_path, _size=1):
    """
    If file size is greater then given _size, create zip of file on same location and
    remove original one. Won't zip If zlib module is not available.
    """
    for root, dirs, files in os.walk(dir_path):
        for _file in files:
            file_path = os.path.join(root, _file)
            size_mb = (old_div(os.path.getsize(file_path), (1024 * 1024)))
            if size_mb >= _size:
                os.chdir(root)
                try:
                    newzip = zipfile.ZipFile(
                        _file + ".zip", "w", zipfile.ZIP_DEFLATED)
                    newzip.write(_file)
                    newzip.close()
                    os.remove(_file)
                except Exception as e:
                    print(e)
                    pass 

def train(config):
    c = Classifier.classifier(config)
    pprint.pprint(c.tuples.all_tuples)
    print('All tuples:',len(c.tuples.all_tuples))
    model_path = config.get("train", "output")
    model_dir = os.path.dirname(model_path)
    if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    print('output to {}'.format(model_path))
    dataroot = config.get("train", "dataroot")
    dataset = config.get("train", "dataset")
    dw = sutils.dataset_walker(dataset = dataset, dataroot=dataroot, labels=True)
    c = Classifier.classifier(config)
    c.cacheFeature(dw)
    c.train(dw)
    c.save(model_path)
    with zipfile.ZipFile(os.path.join(model_dir, 'svm_multiwoz.zip'), 'w', zipfile.ZIP_DEFLATED) as zf:
        zf.write(model_path) 

def get(self, *, tid: objectid.ObjectId):
    tdoc, tsdocs = await contest.get_and_list_status(self.domain_id, document.TYPE_HOMEWORK, tid)
    rnames = {}
    for tsdoc in tsdocs:
      for pdetail in tsdoc.get('detail', []):
        rnames[pdetail['rid']] = 'U{}_P{}_R{}'.format(tsdoc['uid'], pdetail['pid'], pdetail['rid'])
    output_buffer = io.BytesIO()
    zip_file = zipfile.ZipFile(output_buffer, 'a', zipfile.ZIP_DEFLATED)
    rdocs = record.get_multi(get_hidden=True, _id={'$in': list(rnames.keys())})
    async for rdoc in rdocs:
      zip_file.writestr(rnames[rdoc['_id']] + '.' + rdoc['lang'], rdoc['code'])
    # mark all files as created in Windows :p
    for zfile in zip_file.filelist:
      zfile.create_system = 0
    zip_file.close()

    await self.binary(output_buffer.getvalue(), 'application/zip',
                      file_name='{}.zip'.format(tdoc['title'])) 

def create_function_handler_zip(zip_location, main_exec_file, backend_location):

    logger.debug("Creating function handler zip in {}".format(zip_location))

    def add_folder_to_zip(zip_file, full_dir_path, sub_dir=''):
        for file in os.listdir(full_dir_path):
            full_path = os.path.join(full_dir_path, file)
            if os.path.isfile(full_path):
                zip_file.write(full_path, os.path.join('pywren_ibm_cloud', sub_dir, file))
            elif os.path.isdir(full_path) and '__pycache__' not in full_path:
                add_folder_to_zip(zip_file, full_path, os.path.join(sub_dir, file))

    try:
        with zipfile.ZipFile(zip_location, 'w', zipfile.ZIP_DEFLATED) as pywren_zip:
            current_location = os.path.dirname(os.path.abspath(backend_location))
            module_location = os.path.dirname(os.path.abspath(pywren_ibm_cloud.__file__))
            main_file = os.path.join(current_location, 'entry_point.py')
            pywren_zip.write(main_file, main_exec_file)
            add_folder_to_zip(pywren_zip, module_location)

    except Exception:
        raise Exception('Unable to create the {} package: {}'.format(zip_location)) 

def pickle_load(path, compression=False):
    """Unpickle a possible compressed pickle.

    Parameters
    ----------
    path: str
        path to the output file
    compression: bool
        if true assumes that pickle was compressed when created and attempts decompression.

    Returns
    -------
    obj: object
        the unpickled object
    """

    if compression:
        with zipfile.ZipFile(path, "r", compression=zipfile.ZIP_DEFLATED) as myzip:
            with myzip.open("data") as f:
                return pickle.load(f)
    else:
        with open(path, "rb") as f:
            return pickle.load(f) 

def _write_file(self, archive, path: str, fpath: Path) -> None:
        # write content into archive
        archive.write(filename=str(fpath), arcname=path, compress_type=ZIP_DEFLATED)

        # calculate hashsum
        # https://stackoverflow.com/questions/22058048/hashing-a-file-in-python
        digest = sha256()
        with fpath.open('rb') as stream:
            while True:
                data = stream.read(65536)
                if not data:
                    break
                digest.update(data)
        digest = urlsafe_b64encode(digest.digest()).decode().rstrip('=')

        self._records.append((path, digest, fpath.stat().st_size)) 

def savekmz(self, path, format=True):
        """
        Save the kml as a kmz file to the given file supplied by `path`.

        Keyword arguments:
        path (string) -- the path of the kmz file to be saved
        format (bool) -- format the resulting kml "prettyprint" (default True)

        """
        Kmlable._setkmz()
        out = self._genkml(format).encode('utf-8')
        kmz = zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED)
        kmz.writestr("doc.kml", out)
        for image in Kmlable._getimages():
            kmz.write(image, os.path.join('files', os.path.split(image)[1]))
        kmz.close()
        Kmlable._clearimages() 

def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True,
                 mode='w'):
    """Create a zip file from all the files under 'base_dir'.  The output
    zip file will be named 'base_dir' + ".zip".  Uses either the "zipfile"
    Python module (if available) or the InfoZIP "zip" utility (if installed
    and found on the default search path).  If neither tool is available,
    raises DistutilsExecError.  Returns the name of the output zip file.
    """
    import zipfile

    mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
    log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)

    def visit(z, dirname, names):
        for name in names:
            path = os.path.normpath(os.path.join(dirname, name))
            if os.path.isfile(path):
                p = path[len(base_dir) + 1:]
                if not dry_run:
                    z.write(path, p)
                log.debug("adding '%s'", p)

    compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
    if not dry_run:
        z = zipfile.ZipFile(zip_filename, mode, compression=compression)
        for dirname, dirs, files in os.walk(base_dir):
            visit(z, dirname, files)
        z.close()
    else:
        for dirname, dirs, files in os.walk(base_dir):
            visit(None, dirname, files)
    return zip_filename 

def save(self, filename=None):
        """Save the current Mission object to the given file.

        Args:
            filename: filepath to save the Mission object
        """
        filename = self.filename if filename is None else filename
        if not filename:
            raise RuntimeError("No filename given.")
        self.filename = filename  # store filename

        with zipfile.ZipFile(filename, 'w', compression=zipfile.ZIP_DEFLATED) as zipf:
            # options
            zipf.writestr('options', str(self.options))

            # warehouses
            zipf.writestr('warehouses', str(self.warehouses))

            # translation files
            dicttext = lua.dumps(self.translation.dict('DEFAULT'), "dictionary", 1)
            zipf.writestr('l10n/DEFAULT/dictionary', dicttext)

            mapresource = self.map_resource.store(zipf, 'DEFAULT')
            # print(mapresource)
            zipf.writestr('l10n/DEFAULT/mapResource', lua.dumps(mapresource, "mapResource", 1))

            zipf.writestr('mission', str(self))

        return True 

def relatively_safe_pickle_dump(obj, path, compression=False):
    """This is just like regular pickle dump, except from the fact that failure cases are
    different:

        - It's never possible that we end up with a pickle in corrupted state.
        - If a there was a different file at the path, that file will remain unchanged in the
          even of failure (provided that filesystem rename is atomic).
        - it is sometimes possible that we end up with useless temp file which needs to be
          deleted manually (it will be removed automatically on the next function call)

    The indended use case is periodic checkpoints of experiment state, such that we never
    corrupt previous checkpoints if the current one fails.

    Parameters
    ----------
    obj: object
        object to pickle
    path: str
        path to the output file
    compression: bool
        if true pickle will be compressed
    """
    temp_storage = path + ".relatively_safe"
    if compression:
        # Using gzip here would be simpler, but the size is limited to 2GB
        with tempfile.NamedTemporaryFile() as uncompressed_file:
            pickle.dump(obj, uncompressed_file)
            uncompressed_file.file.flush()
            with zipfile.ZipFile(temp_storage, "w", compression=zipfile.ZIP_DEFLATED) as myzip:
                myzip.write(uncompressed_file.name, "data")
    else:
        with open(temp_storage, "wb") as f:
            pickle.dump(obj, f)
    os.rename(temp_storage, path) 

def _get_archive_writer(output_path):
    """
    Detects and returns the appropriate archive writer

    output_path is the pathlib.Path of the archive to write
    """
    if not output_path.suffixes:
        raise ValueError('Output name has no suffix: %s' % output_path.name)
    if output_path.suffixes[-1].lower() == '.zip':
        import zipfile
        archive_root = Path(output_path.stem)
        output_archive = zipfile.ZipFile(str(output_path), 'w', zipfile.ZIP_DEFLATED)

        def add_func(in_path, arc_path):
            """Add files to zip archive"""
            if in_path.is_dir():
                for sub_path in in_path.rglob('*'):
                    output_archive.write(
                        str(sub_path), str(arc_path / sub_path.relative_to(in_path)))
            else:
                output_archive.write(str(in_path), str(arc_path))
    elif '.tar' in output_path.name.lower():
        import tarfile
        if len(output_path.suffixes) >= 2 and output_path.suffixes[-2].lower() == '.tar':
            tar_mode = 'w:%s' % output_path.suffixes[-1][1:]
            archive_root = Path(output_path.with_suffix('').stem)
        elif output_path.suffixes[-1].lower() == '.tar':
            tar_mode = 'w'
            archive_root = Path(output_path.stem)
        else:
            raise ValueError('Could not detect tar format for output: %s' % output_path.name)
        output_archive = tarfile.open(str(output_path), tar_mode)
        add_func = lambda in_path, arc_path: output_archive.add(str(in_path), str(arc_path))
    else:
        raise ValueError('Unknown archive extension with name: %s' % output_path.name)
    return output_archive, add_func, archive_root 

def compress_kml(kml):
    "Returns compressed KMZ from the given KML string."
    kmz = BytesIO()
    zf = zipfile.ZipFile(kmz, 'a', zipfile.ZIP_DEFLATED)
    zf.writestr('doc.kml', kml.encode(settings.DEFAULT_CHARSET))
    zf.close()
    kmz.seek(0)
    return kmz.read() 

def zip_directory(src_dir, dst):
        assert os.path.isdir(src_dir), "Invalid, not a directory: '{}'".format(src_dir)
        with Archive._open["zip"](dst, 'w', zipfile.ZIP_DEFLATED) as write_handler:
            Archive.zip_stream(src_dir, write_handler) 

def build(self):
        if self._clean_build:
            self._prepare_clean_build()

        self._run_hooks(self._hooks.get('pre', []))

        with PyZipFile(self._zippath, 'w', compression=ZIP_DEFLATED) as zipfile:
            for p in self._get_paths():
                if os.path.isdir(p):
                    self._archive_dir(zipfile, p)
                else:
                    self._archive_file(zipfile, p)

            if not self._single_file:
                secret_path = os.path.join(self._tmpdir, lamvery.secret.SECRET_FILE_NAME)
                env_path = os.path.join(self._tmpdir, lamvery.env.ENV_FILE_NAME)
                self._generate_json(secret_path, self._secret)
                self._generate_json(env_path, self._env)
                self._archive_file(zipfile, secret_path)
                self._archive_file(zipfile, env_path)

                if self._runtime == lamvery.config.RUNTIME_NODE_JS:
                    self._archive_dist(zipfile, 'lamvery.js')

        self._run_hooks(self._hooks.get('post', []))

        return open(self._zippath, 'rb') 

def test_build_with_single_file(self):
        builder = Builder('test.zip', function_filename='lambda_function.py', single_file=True)
        builder.build()
        with PyZipFile(builder._zippath, 'r', compression=ZIP_DEFLATED) as zipfile:
            ok_('lambda_function.py' in zipfile.namelist())
            ok_(not ('.lamvery_secret.json' in zipfile.namelist())) 

def test_build(self):
        builder = Builder('test.zip')
        builder._runtime = RUNTIME_NODE_JS
        ok_(hasattr(builder.build(), 'read'))
        with PyZipFile(builder._zippath, 'r', compression=ZIP_DEFLATED) as zipfile:
            ok_('lambda_function.pyc' in zipfile.namelist())
            ok_('.lamvery_secret.json' in zipfile.namelist()) 

def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True,
                 mode='w'):
    """Create a zip file from all the files under 'base_dir'.  The output
    zip file will be named 'base_dir' + ".zip".  Uses either the "zipfile"
    Python module (if available) or the InfoZIP "zip" utility (if installed
    and found on the default search path).  If neither tool is available,
    raises DistutilsExecError.  Returns the name of the output zip file.
    """
    import zipfile

    mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
    log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)

    def visit(z, dirname, names):
        for name in names:
            path = os.path.normpath(os.path.join(dirname, name))
            if os.path.isfile(path):
                p = path[len(base_dir) + 1:]
                if not dry_run:
                    z.write(path, p)
                log.debug("adding '%s'" % p)

    compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
    if not dry_run:
        z = zipfile.ZipFile(zip_filename, mode, compression=compression)
        for dirname, dirs, files in os.walk(base_dir):
            visit(z, dirname, files)
        z.close()
    else:
        for dirname, dirs, files in os.walk(base_dir):
            visit(None, dirname, files)
    return zip_filename 

def build_zip(self, pathname, archive_paths):
        with ZipFile(pathname, 'w', zipfile.ZIP_DEFLATED) as zf:
            for ap, p in archive_paths:
                logger.debug('Wrote %s to %s in wheel', p, ap)
                zf.write(p, ap) 

def __init__(self):
        self.in_memory_data = io.BytesIO()
        self.in_memory_zip = zipfile.ZipFile(
            self.in_memory_data, "w", zipfile.ZIP_DEFLATED, False)
        self.in_memory_zip.debug = 3 

def _zip_file(self, file_dir: str, market_id: str) -> str:
        """zips txt file into filename.zip
        """
        zip_file_directory = os.path.join(
            self.local_dir, self.recorder_id, "%s.zip" % market_id
        )
        with zipfile.ZipFile(zip_file_directory, mode="w") as zf:
            zf.write(
                file_dir, os.path.basename(file_dir), compress_type=zipfile.ZIP_DEFLATED
            )
        return zip_file_directory 

def _zip_to_model(tmpdir, model_zip_path):
    # make pickle file with model options
    # create zipfile archive
    zf = zipfile.ZipFile(model_zip_path, 'w', allowZip64=True)
    zf.compress_type = zipfile.ZIP_DEFLATED  # saw a note that this helps with backwards compat

    # Adding files from directory 'files'
    for _, _, files in os.walk(tmpdir):
        for f in files:
            zf.write(os.path.join(tmpdir, f), f)