Python zlib.crc32() Examples

def test_img_cdn_hard_rewrite(self):
        """测试重写html中CDN的链接 https://httpbin.org/"""
        # 第一次请求, 没有使用CDN
        self.rv = self.client.get(
            self.url("/image/jpeg"),
            environ_base=env(),
            headers=headers()
        )  # type: Response

        # 由于flaks的惰性, 需要先实际获取一次结果, 缓存才能实际被存储生效
        self.assertEqual("image/jpeg", self.rv.content_type, msg=self.dump())
        self.assertEqual(200, self.rv.status_code, msg=self.dump())
        self.assertEqual(0x97ca823f, crc32(self.rv.data), msg=self.dump())

        with self.app.test_client() as c:
            # 请求包含 https://httpbin.org/image/jpeg 的页面, 其中这张图片的链接会被重写成CDN
            self.rv2 = c.get(
                self.url("/base64/PGltZyBzcmM9Imh0dHBzOi8vaHR0cGJpbi5vcmcvaW1hZ2UvanBlZyI+Cg=="),
                environ_base=env(),
                headers=headers()
            )  # type: Response
            self.assertIn(b"cdn2.zmirror-unittest.com/image/jpeg", self.rv2.data, msg=self.dump()) 

def write(self,data):
        self._check_closed()
        if self.mode != WRITE:
            import errno
            raise IOError(errno.EBADF, "write() on read-only GzipFile object")

        if self.fileobj is None:
            raise ValueError, "write() on closed GzipFile object"

        # Convert data type if called by io.BufferedWriter.
        if isinstance(data, memoryview):
            data = data.tobytes()

        if len(data) > 0:
            self.fileobj.write(self.compress.compress(data))
            self.size += len(data)
            self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
            self.offset += len(data)

        return len(data) 

def _read_eof(self):
        # We've read to the end of the file, so we have to rewind in order
        # to reread the 8 bytes containing the CRC and the file size.
        # We check the that the computed CRC and size of the
        # uncompressed data matches the stored values.  Note that the size
        # stored is the true file size mod 2**32.
        self.fileobj.seek(-8, 1)
        crc32 = read32(self.fileobj)
        isize = read32(self.fileobj)  # may exceed 2GB
        if crc32 != self.crc:
            raise IOError("CRC check failed %s != %s" % (hex(crc32),
                                                         hex(self.crc)))
        elif isize != (self.size & 0xffffffffL):
            raise IOError, "Incorrect length of data produced"

        # Gzip files can be padded with zeroes and still have archives.
        # Consume all zero bytes and set the file position to the first
        # non-zero byte. See http://www.gzip.org/#faq8
        c = "\x00"
        while c == "\x00":
            c = self.fileobj.read(1)
        if c:
            self.fileobj.seek(-1, 1) 

def get_files_types(self):
        """
        Return the files inside the APK with their associated types (by using python-magic)

        :rtype: a dictionnary
        """
        if self._files == {}:
            # Generate File Types / CRC List
            for i in self.get_files():
                buffer = self.zip.read(i)
                self.files_crc32[i] = crc32(buffer)
                # FIXME why not use the crc from the zipfile?
                # should be validated as well.
                # crc = self.zip.getinfo(i).CRC
                self._files[i] = self._get_file_magic_name(buffer)

        return self._files 

def computehash((filedir, filename, extrahashes, sha256sum)):
	resolved_path = os.path.join(filedir, filename)
	filehashes = {}
	filehashes['sha256'] = sha256sum
	scanfile = open(resolved_path, 'r')
	data = scanfile.read()
	scanfile.close()
	for i in extrahashes:
		if i == 'crc32':
			filehashes[i] = zlib.crc32(data) & 0xffffffff
		elif i == 'tlsh':
			if os.stat(resolved_path).st_size >= 256:
				tlshhash = tlsh.hash(data)
				filehashes[i] = tlshhash
			else:
				filehashes[i] = None
		else:
			h = hashlib.new(i)
			h.update(data)
			filehashes[i] = h.hexdigest()
	filehashes['sha256'] = sha256sum
	return (filedir, filename, filehashes) 

def __init__(self, seed):
        if not seed:
            seed = "%.1f" % time.time()

        if hasattr(seed, "encode"):
            seed = seed.encode('ascii')

        # A note on hashfunctions.
        # We don't need cryptographic quality, so we won't use hashlib -
        # that'd be way to slow. The zlib module contains two hash
        # functions. Adler32 is fast, but not very uniform for short
        # strings. Crc32 is slower, but has better bit distribution.
        # So, we use crc32 whenever the hash is converted into an
        # exportable number, but we use adler32 when we're producing
        # intermediate values.
        self.seed = zlib.adler32(seed)
        self.text_seed = seed

        # Default, typically overridden
        self.size = 1024 + 786j 

def gzip_app_iter(app_iter):
    size = 0
    crc = zlib.crc32(b"") & 0xffffffff
    compress = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS,
                                zlib.DEF_MEM_LEVEL, 0)

    yield _gzip_header
    for item in app_iter:
        size += len(item)
        crc = zlib.crc32(item, crc) & 0xffffffff

        # The compress function may return zero length bytes if the input is
        # small enough; it buffers the input for the next iteration or for a
        # flush.
        result = compress.compress(item)
        if result:
            yield result

    # Similarly, flush may also not yield a value.
    result = compress.flush()
    if result:
        yield result
    yield struct.pack("<2L", crc, size & 0xffffffff) 

def uniform(self, key, min_value=0., max_value=1.):
        """Returns a random number between min_value and max_value"""
        return min_value + self._random(key) * (max_value - min_value)

    # def complex(self, key):
    #     """Returns a random complex number.

    #     Both real and imaginary component are between 0 and 1 (inclusive)"""

    #     if hasattr(key, "encode"):
    #         key.encode('ascii')

    #     value1 = zlib.crc32(key, self.seed) & MAX_VALUE
    #     value2 = zlib.crc32(key, value1) & MAX_VALUE

    #     return (float(value1) + 1j * float(value2)) * INV_MAX_VALUE 

def __init__(self, *args):
        super().__init__(*args)
        self.packsize   = None      # uint32|64 packed size
        self.origsize   = None      # uint32|64 original size
        self.datetime   = None      # uint32    ctime
        self.attribs    = None      # uint32    file attributes
        self.crc32      = None      # uint32    checksum over compressed file
        self.comptype   = None      # uint8     compression type
        self.compqual   = None      # uint8     compression quality
        self.params     = None      # uint16    decompression parameters
        self.reserved1  = None      # uint16
        self.filename   = None      # [uint16]
        self.comment    = b''       # [uint16]  optional, compressed
        self.ntsecurity = b''       # [uint16]  optional
        self.reserved2  = None      # ?
        self.dataoffset = None      #           position of data after hdr 

def via_annotation_stream(uri):
    name = threading.currentThread().name
    start = time.time()
    total_size = 0
    print("thread {0} downloading via annotation stream...".format(name))
    with Proxy(uri) as p:
        perform_checksum = False
        for progress, checksum in p.annotation_stream(perform_checksum):
            chunk = current_context.response_annotations["FDAT"]
            if perform_checksum and zlib.crc32(chunk) != checksum:
                raise ValueError("checksum error")
            total_size += len(chunk)
            assert progress == total_size
            current_context.response_annotations.clear()  # clean them up once we're done with them
    duration = time.time() - start
    print("thread {0} done, {1:.2f} Mb/sec.".format(name, total_size/1024.0/1024.0/duration)) 

def annotation_stream(self, with_checksum=False):
        # create a large temporary file
        f = tempfile.TemporaryFile()
        for _ in range(5000):
            f.write(b"1234567890!" * 1000)
        filesize = f.tell()
        f.seek(os.SEEK_SET, 0)
        # return the file data via annotation stream (remote iterator)
        annotation_size = 500000
        print("transmitting file via annotations stream (%d bytes in chunks of %d)..." % (filesize, annotation_size))
        with f:
            while True:
                chunk = f.read(annotation_size)
                if not chunk:
                    break
                # store the file data chunk in the FDAT response annotation,
                # and return the current file position and checksum (if asked).
                current_context.response_annotations = {"FDAT": chunk}
                yield f.tell(), zlib.crc32(chunk) if with_checksum else 0 

def write(self,data):
        self._check_closed()
        if self.mode != WRITE:
            import errno
            raise IOError(errno.EBADF, "write() on read-only GzipFile object")

        if self.fileobj is None:
            raise ValueError, "write() on closed GzipFile object"

        # Convert data type if called by io.BufferedWriter.
        if isinstance(data, memoryview):
            data = data.tobytes()

        if len(data) > 0:
            self.size = self.size + len(data)
            self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
            self.fileobj.write( self.compress.compress(data) )
            self.offset += len(data)

        return len(data) 

def _read_eof(self):
        # We've read to the end of the file, so we have to rewind in order
        # to reread the 8 bytes containing the CRC and the file size.
        # We check the that the computed CRC and size of the
        # uncompressed data matches the stored values.  Note that the size
        # stored is the true file size mod 2**32.
        self.fileobj.seek(-8, 1)
        crc32 = read32(self.fileobj)
        isize = read32(self.fileobj)  # may exceed 2GB
        if crc32 != self.crc:
            raise IOError("CRC check failed %s != %s" % (hex(crc32),
                                                         hex(self.crc)))
        elif isize != (self.size & 0xffffffffL):
            raise IOError, "Incorrect length of data produced"

        # Gzip files can be padded with zeroes and still have archives.
        # Consume all zero bytes and set the file position to the first
        # non-zero byte. See http://www.gzip.org/#faq8
        c = "\x00"
        while c == "\x00":
            c = self.fileobj.read(1)
        if c:
            self.fileobj.seek(-1, 1) 

def _get_crc32(self, filename):
        """
        Calculates and compares the CRC32 and returns the raw buffer.

        The CRC32 is added to `files_crc32` dictionary, if not present.

        :param filename: filename inside the zipfile
        :rtype: bytes
        """
        buffer = self.zip.read(filename)
        if filename not in self.files_crc32:
            self.files_crc32[filename] = crc32(buffer)
            if self.files_crc32[filename] != self.zip.getinfo(filename).CRC:
                log.error("File '{}' has different CRC32 after unpacking! "
                          "Declared: {:08x}, Calculated: {:08x}".format(filename,
                                                                        self.zip.getinfo(filename).CRC,
                                                                        self.files_crc32[filename]))
        return buffer 

def __init__(self, idx, filehdrs, f):
        """
        Initialize an :class:`AceMember` object with index within archive *idx*,
        initial file header *filehdr* and underlying file-like object *f*.
        """
        self._idx           = idx
        self._file          = f
        self._headers       = filehdrs
        self.__attribs      = filehdrs[0].attribs
        self.__comment      = filehdrs[0].comment.decode('utf-8',
                                                         errors='replace')
        self.__crc32        = filehdrs[-1].crc32
        self.__comptype     = filehdrs[0].comptype
        self.__compqual     = filehdrs[0].compqual
        self.__datetime     = _dt_fromdos(filehdrs[0].datetime)
        self.__dicsizebits  = (filehdrs[0].params & 15) + 10
        self.__dicsize      = 1 << self.__dicsizebits
        self.__raw_filename = filehdrs[0].filename
        self.__filename     = self._sanitize_filename(filehdrs[0].filename)
        if self.__filename == '':
            self.__filename = 'file%04i' % self._idx
        self.__ntsecurity   = filehdrs[0].ntsecurity
        self.__size         = filehdrs[0].origsize
        self.__packsize     = 0
        for hdr in filehdrs:
            self.__packsize += hdr.packsize 

def encode(buf, width, height):
  """ buf: must be bytes or a bytearray in py3, a regular string in py2. formatted RGBRGB... """
  assert (width * height * 3 == len(buf))
  bpp = 3

  def raw_data():
    # reverse the vertical line order and add null bytes at the start
    row_bytes = width * bpp
    for row_start in range((height - 1) * width * bpp, -1, -row_bytes):
      yield b'\x00'
      yield buf[row_start:row_start + row_bytes]

  def chunk(tag, data):
    return [
        struct.pack("!I", len(data)),
        tag,
        data,
        struct.pack("!I", 0xFFFFFFFF & zlib.crc32(data, zlib.crc32(tag)))
      ]

  SIGNATURE = b'\x89PNG\r\n\x1a\n'
  COLOR_TYPE_RGB = 2
  COLOR_TYPE_RGBA = 6
  bit_depth = 8
  return b''.join(
      [ SIGNATURE ] +
      chunk(b'IHDR', struct.pack("!2I5B", width, height, bit_depth, COLOR_TYPE_RGB, 0, 0, 0)) +
      chunk(b'IDAT', zlib.compress(b''.join(raw_data()), 9)) +
      chunk(b'IEND', b'')
    ) 

def write(self, s):
        """Write string s to the stream.
        """
        if self.comptype == "gz":
            self.crc = self.zlib.crc32(s, self.crc)
        self.pos += len(s)
        if self.comptype != "tar":
            s = self.cmp.compress(s)
        self.__write(s) 

def write(self, s):
        """Write string s to the stream.
        """
        if self.comptype == "gz":
            self.crc = self.zlib.crc32(s, self.crc)
        self.pos += len(s)
        if self.comptype != "tar":
            s = self.cmp.compress(s)
        self.__write(s) 

def __iadd__(self, buf):
        """
        Adding a buffer of bytes into the checksum, updating the rolling
        checksum from all previously added buffers.
        """
        self.__state = zlib.crc32(buf, self.__state)
        return self 

def __str__(self):
        return super().__str__() + """
    packsize    %i
    origsize    %i
    datetime    0x%08x  %s
    attribs     0x%08x  %s
    crc32       0x%08x
    comptype    0x%02x        %s
    compqual    0x%02x        %s
    params      0x%04x
    reserved1   0x%04x
    filename    %r
    comment     %r
    ntsecurity  %r
    reserved2   %r""" % (
                self.packsize,
                self.origsize,
                self.datetime,
                _dt_fromdos(self.datetime).strftime('%Y-%m-%d %H:%M:%S'),
                self.attribs, self.attribs_str,
                self.crc32,
                self.comptype, self.comptype_str,
                self.compqual, self.compqual_str,
                self.params,
                self.reserved1,
                self.filename,
                self.comment,
                self.ntsecurity,
                self.reserved2) 

def write(self, s):
        """Write string s to the stream.
        """
        if self.comptype == "gz":
            self.crc = self.zlib.crc32(s, self.crc)
        self.pos += len(s)
        if self.comptype != "tar":
            s = self.cmp.compress(s)
        self.__write(s) 

def get_files_crc32(self):
        """
        Calculates and returns a dictionary of filenames and CRC32

        :return: dict of filename: CRC32
        """
        if self.files_crc32 == {}:
            for i in self.get_files():
                buffer = self.zip.read(i)
                self.files_crc32[i] = crc32(buffer)

        return self.files_crc32 

def encode(buf, width, height):
  """ buf: must be bytes or a bytearray in py3, a regular string in py2. formatted RGBRGB... """
  assert (width * height * 3 == len(buf))
  bpp = 3

  def raw_data():
    # reverse the vertical line order and add null bytes at the start
    row_bytes = width * bpp
    for row_start in range((height - 1) * width * bpp, -1, -row_bytes):
      yield b'\x00'
      yield buf[row_start:row_start + row_bytes]

  def chunk(tag, data):
    return [
        struct.pack("!I", len(data)),
        tag,
        data,
        struct.pack("!I", 0xFFFFFFFF & zlib.crc32(data, zlib.crc32(tag)))
      ]

  SIGNATURE = b'\x89PNG\r\n\x1a\n'
  COLOR_TYPE_RGB = 2
  COLOR_TYPE_RGBA = 6
  bit_depth = 8
  return b''.join(
      [ SIGNATURE ] +
      chunk(b'IHDR', struct.pack("!2I5B", width, height, bit_depth, COLOR_TYPE_RGB, 0, 0, 0)) +
      chunk(b'IDAT', zlib.compress(b''.join(raw_data()), 9)) +
      chunk(b'IEND', b'')
    ) 

def _init_read(self):
        self.crc = zlib.crc32("") & 0xffffffffL
        self.size = 0 

def _init_write(self, filename):
        self.name = filename
        self.crc = zlib.crc32("") & 0xffffffffL
        self.size = 0
        self.writebuf = []
        self.bufsize = 0 

def write(self, s):
        """Write string s to the stream.
        """
        if self.comptype == "gz":
            self.crc = self.zlib.crc32(s, self.crc) & 0xffffffffL
        self.pos += len(s)
        if self.comptype != "tar":
            s = self.cmp.compress(s)
        self.__write(s) 

def _validate_file_index(index_file, resolved_tree, cache_index_files):
    """
    Validation of file index and hashes against the source tree.
        Updates cache_index_files

    Returns True if the file index is valid; False otherwise
    """
    all_hashes_valid = True
    crc32_regex = re.compile(r'^[a-zA-Z0-9]{8}$')
    for entry in index_file.read().decode(ENCODING).splitlines():
        try:
            relative_path, file_hash = entry.split(_INDEX_HASH_DELIMITER)
        except ValueError as exc:
            get_logger().error('Could not split entry "%s": %s', entry, exc)
            continue
        if not relative_path or not file_hash:
            get_logger().error('Entry %s of domain substitution cache file index is not valid',
                               _INDEX_HASH_DELIMITER.join((relative_path, file_hash)))
            all_hashes_valid = False
            continue
        if not crc32_regex.match(file_hash):
            get_logger().error('File index hash for %s does not appear to be a CRC32 hash',
                               relative_path)
            all_hashes_valid = False
            continue
        if zlib.crc32((resolved_tree / relative_path).read_bytes()) != int(file_hash, 16):
            get_logger().error('Hashes do not match for: %s', relative_path)
            all_hashes_valid = False
            continue
        if relative_path in cache_index_files:
            get_logger().error('File %s shows up at least twice in the file index', relative_path)
            all_hashes_valid = False
            continue
        cache_index_files.add(relative_path)
    return all_hashes_valid 

def _substitute_path(path, regex_iter):
    """
    Perform domain substitution on path and add it to the domain substitution cache.

    path is a pathlib.Path to the file to be domain substituted.
    regex_iter is an iterable of regular expression namedtuple like from
        config.DomainRegexList.regex_pairs()

    Returns a tuple of the CRC32 hash of the substituted raw content and the
        original raw content; None for both entries if no substitutions were made.

    Raises FileNotFoundError if path does not exist.
    Raises UnicodeDecodeError if path's contents cannot be decoded.
    """
    with path.open('r+b') as input_file:
        original_content = input_file.read()
        if not original_content:
            return (None, None)
        content = None
        encoding = None
        for encoding in TREE_ENCODINGS:
            try:
                content = original_content.decode(encoding)
                break
            except UnicodeDecodeError:
                continue
        if not content:
            raise UnicodeDecodeError('Unable to decode with any encoding: %s' % path)
        file_subs = 0
        for regex_pair in regex_iter:
            content, sub_count = regex_pair.pattern.subn(regex_pair.replacement, content)
            file_subs += sub_count
        if file_subs > 0:
            substituted_content = content.encode(encoding)
            input_file.seek(0)
            input_file.write(content.encode(encoding))
            input_file.truncate()
            return (zlib.crc32(substituted_content), original_content)
        return (None, None) 

def get_crc32(raf, start=0):
    raf.seek(0)
    crc = 0
    while True:
        buffer = raf.read(128 * 1024)
        if len(buffer) == 0:
            return reverse32(crc & MASK)
        else:
            crc = zlib.crc32(buffer, start) 

def write(self):
        """Return a string encoding of the page header and data.

        A ValueError is raised if the data is too big to fit in a
        single page.
        """

        data = [
            struct.pack("<4sBBqIIi", b"OggS", self.version, self.__type_flags,
                        self.position, self.serial, self.sequence, 0)
        ]

        lacing_data = []
        for datum in self.packets:
            quot, rem = divmod(len(datum), 255)
            lacing_data.append(b"\xff" * quot + chr_(rem))
        lacing_data = b"".join(lacing_data)
        if not self.complete and lacing_data.endswith(b"\x00"):
            lacing_data = lacing_data[:-1]
        data.append(chr_(len(lacing_data)))
        data.append(lacing_data)
        data.extend(self.packets)
        data = b"".join(data)

        # Python's CRC is swapped relative to Ogg's needs.
        # crc32 returns uint prior to py2.6 on some platforms, so force uint
        crc = (~zlib.crc32(data.translate(cdata.bitswap), -1)) & 0xffffffff
        # Although we're using to_uint_be, this actually makes the CRC
        # a proper le integer, since Python's CRC is byteswapped.
        crc = cdata.to_uint_be(crc).translate(cdata.bitswap)
        data = data[:22] + crc + data[26:]
        return data