Python typing.ByteString() Examples

def decompress(bs: ByteString, size: int) -> ByteString:
    bs = bytearray(bs)
    out = bytearray()
    blen = len(bs)
    pos = 0
    while pos < blen and len(out) < size:
        control = bs[pos]
        pos += 1
        for i in range(8):
            b = control & (1 << i) > 0
            if not pos < blen:
                break
            if not b:
                out.append(bs[pos])
                pos += 1
            else:
                length = (bs[pos] >> 2) + 3
                distance = (bs[pos] & 0b11) << 8 | bs[pos+1]
                pos += 2
                backref = out[-distance:]
                lookup = backref * int(length / distance) + backref[:(length % distance)]
                out += lookup
    return out[:size] 

def write_bytes_buf(data: ByteString) -> bytearray:
    return num_to_varint(len(data)) + data 

def read_bytes_from_buf(data: ByteString, offset) -> Tuple[bytearray, int]:
    data_len, offset = read_varint_from_buf(data, offset)
    if offset + data_len >= len(data):
        raise ValueError('Corrupted data found.')
    return data[offset: offset + data_len], offset + data_len 

def add_leaf(self, values: Union[Iterable[ByteString], ByteString], do_hash=False):
        self.is_ready = False
        # check if single leaf
        if not isinstance(values, Iterable):
            values = [values]

        for v in values:
            if do_hash:
                v = self.hash_function(v).digest()
            v = bytes(v)
            self.leaves.append(v) 

def test_bytestring(self):
        assert isinstance(b'', typing.ByteString)
        assert isinstance(bytearray(b''), typing.ByteString) 

def read(self) -> ByteString:
        return json.dumps(self.response).encode() 

def test_bytestring(self):
        self.assertIsInstance(b'', typing.ByteString)
        self.assertIsInstance(bytearray(b''), typing.ByteString) 

def blob2numpy(value: ByteString, shape: Union[list, tuple], dtype: str) -> np.ndarray:
    """Convert `BLOB` result from RedisAI to `np.ndarray`."""
    mm = {
        'FLOAT': 'float32',
        'DOUBLE': 'float64'
    }
    dtype = mm.get(dtype, dtype.lower())
    a = np.frombuffer(value, dtype=dtype)
    return a.reshape(shape) 

def modelset(name: AnyStr, backend: str, device: str, data: ByteString,
             batch: int, minbatch: int, tag: AnyStr,
             inputs: Union[AnyStr, List[AnyStr]],
             outputs: Union[AnyStr, List[AnyStr]]) -> Sequence:
    if device.upper() not in utils.allowed_devices:
        raise ValueError(f"Device not allowed. Use any from {utils.allowed_devices}")
    if backend.upper() not in utils.allowed_backends:
        raise ValueError(f"Backend not allowed. Use any from {utils.allowed_backends}")
    args = ['AI.MODELSET', name, backend, device]

    if batch is not None:
        args += ['BATCHSIZE', batch]
    if minbatch is not None:
        args += ['MINBATCHSIZE', minbatch]
    if tag is not None:
        args += ['TAG', tag]

    if backend.upper() == 'TF':
        if not(all((inputs, outputs))):
            raise ValueError(
                'Require keyword arguments input and output for TF models')
        args += ['INPUTS', *utils.listify(inputs)]
        args += ['OUTPUTS', *utils.listify(outputs)]
    chunk_size = 500 * 1024 * 1024
    data_chunks = [data[i:i + chunk_size] for i in range(0, len(data), chunk_size)]
    # TODO: need a test case for this
    args += ['BLOB', *data_chunks]
    return args 

def _decode_header_data(data: ByteString) -> Tuple[int, int]:
    header_data = rlp.decode(data, sedes=HEADER_DATA_SEDES, strict=False)
    return header_data 

def prepare_hw_encryption_attrs(hw_session: HwSessionInfo, label: str) -> \
        Tuple[int, int, List[int], ByteString, ByteString, ByteString]:
    """

    :param hw_session:
    :param label:
    :return: 0: protocol id
             1: hw type id
             1: hw passphrase encoding
             2: hw bip32 path usad to encodind
    """
    # generate a new random password which will be used to encrypt with Trezor method + Fernet
    protocol = 1
    hw_type_bin = {
            HWType.trezor: 1,
            HWType.keepkey: 2,
            HWType.ledger_nano_s: 3
        }[hw_session.hw_type]

    key = Fernet.generate_key()  # encryption key
    key_bin = base64.urlsafe_b64decode(key)

    bip32_path_n = [10, 100, 1000]

    if hw_session.hw_type in (HWType.trezor, HWType.keepkey):
        # for trezor method, for encryption we use the raw key and the key encrypted with a device
        # will be part of a header
        encrypted_key_bin, pub_key = hw_encrypt_value(hw_session, bip32_path_n, label=label, value=key_bin)
        pub_key_hash = SHA256.new(pub_key).digest()
        return (protocol, hw_type_bin, bip32_path_n, key, encrypted_key_bin, pub_key_hash)

    elif hw_session.hw_type == HWType.ledger_nano_s:
        # Ledger Nano S does not have encryption/decryption features, so for encryption and decryptionwe will use
        # a hash of a signed message, where the message the raw key itself;
        # The raw key will be part of the encrypted header.

        display_label = f'<b>Click the sign message confirmation button on the <br>hardware wallet to encrypt \'{label}\'.</b>'
        bip32_path_str = bip32_path_n_to_string(bip32_path_n)
        sig = hw_sign_message(hw_session, bip32_path_str, key_bin.hex(), display_label=display_label)
        adr_pk = get_address_and_pubkey(hw_session, bip32_path_str)

        pub_key_hash = SHA256.new(adr_pk.get('publicKey')).digest()
        enc_key_hash = SHA256.new(sig.signature).digest()
        enc_key_hash = base64.urlsafe_b64encode(enc_key_hash)

        return (protocol, hw_type_bin, bip32_path_n, enc_key_hash, key_bin, pub_key_hash) 

def hw_encrypt_value(hw_session: HwSessionInfo, bip32_path_n: List[int], label: str,
                     value: ByteString, ask_on_encrypt=True, ask_on_decrypt=True) -> Tuple[bytearray, bytearray]:
    """Encrypts a value with a hardware wallet.
    :param hw_session:
    :param bip32_path_n: bip32 path of the private key used for encryption
    :param label: key (in the meaning of key-value) used for encryption
    :param value: value being encrypted
    :param ask_on_encrypt: see Trezor doc
    :param ask_on_decrypt: see Trezor doc
    """

    def encrypt(ctrl, hw_session: HwSessionInfo, bip32_path_n: List[int], label: str,
                value: bytearray):
        ctrl.dlg_config_fun(dlg_title="Data encryption", show_progress_bar=False)
        ctrl.display_msg_fun(f'<b>Encrypting \'{label}\'...</b>'
                             f'<br><br>Enter the hardware wallet PIN/passphrase (if needed) to encrypt data.<br><br>'
                             f'<b>Note:</b> encryption passphrase is independent from the wallet passphrase  <br>'
                             f'and can vary for each encrypted file.')

        if hw_session.hw_type == HWType.trezor:
            from trezorlib import misc, btc
            from trezorlib import exceptions

            try:
                client = hw_session.hw_client
                data = misc.encrypt_keyvalue(client, bip32_path_n, label, value, ask_on_encrypt, ask_on_decrypt)
                pub_key = btc.get_public_node(client, bip32_path_n).node.public_key
                return data, pub_key
            except (CancelException, exceptions.Cancelled):
                raise CancelException()

        elif hw_session.hw_type == HWType.keepkey:

            client = hw_session.hw_client
            data = client.encrypt_keyvalue(bip32_path_n, label, value, ask_on_encrypt, ask_on_decrypt)
            pub_key = client.get_public_node(bip32_path_n).node.public_key
            return data, pub_key

        elif hw_session.hw_type == HWType.ledger_nano_s:

            raise Exception('Feature not available for Ledger Nano S.')

        else:
            raise Exception('Invalid HW type: ' + str(hw_session))

    if len(value) != 32:
        raise ValueError("Invalid password length (<> 32).")

    return WndUtils.run_thread_dialog(encrypt, (hw_session, bip32_path_n, label, value), True,
                                      force_close_dlg_callback=partial(cancel_hw_thread_dialog, hw_session.hw_client),
                                      show_window_delay_ms=200) 

def hw_decrypt_value(hw_session: HwSessionInfo, bip32_path_n: List[int], label: str,
                     value: ByteString, ask_on_encrypt=True, ask_on_decrypt=True) -> Tuple[bytearray, bytearray]:
    """
    :param hw_session:
    :param passphrase_encoding: (for Keepkey only) it allows forcing the passphrase encoding compatible with BIP-39
        standard (NFKD), which is used by Trezor devices; by default Keepkey uses non-standard encoding (NFC).
    :param bip32_path_n: bip32 path of the private key used for encryption
    :param label: key (in the meaning of key-value) used for encryption
    :param value: encrypted value to be decrypted,
    :param ask_on_encrypt: see Trezor doc
    :param ask_on_decrypt: see Trezor doc
    """

    def decrypt(ctrl, hw_session: HwSessionInfo, bip32_path_n: List[int], label: str, value: bytearray):
        ctrl.dlg_config_fun(dlg_title="Data decryption", show_progress_bar=False)
        ctrl.display_msg_fun(f'<b>Decrypting \'{label}\'...</b><br><br>Enter the hardware wallet PIN/passphrase '
                             f'(if needed)<br> and click the confirmation button to decrypt data.')

        if hw_session.hw_type == HWType.trezor:

            from trezorlib import misc, btc
            from trezorlib import exceptions

            try:
                client = hw_session.hw_client
                data = misc.decrypt_keyvalue(client, bip32_path_n, label, value, ask_on_encrypt, ask_on_decrypt)
                pub_key = btc.get_public_node(client, bip32_path_n).node.public_key
                return data, pub_key
            except (CancelException, exceptions.Cancelled):
                raise CancelException()

        elif hw_session.hw_type == HWType.keepkey:

            client = hw_session.hw_client
            data = client.decrypt_keyvalue(bip32_path_n, label, value, ask_on_encrypt, ask_on_decrypt)
            pub_key = client.get_public_node(bip32_path_n).node.public_key
            return data, pub_key

        elif hw_session.hw_type == HWType.ledger_nano_s:

            raise Exception('Feature not available for Ledger Nano S.')

        else:
            raise Exception('Invalid HW type: ' + str(hw_session))

    if len(value) != 32:
        raise ValueError("Invalid password length (<> 32).")

    return WndUtils.run_thread_dialog(decrypt, (hw_session, bip32_path_n, label, value), True,
                                      force_close_dlg_callback=partial(cancel_hw_thread_dialog, hw_session.hw_client)) 

def modelset(self,
                 key: AnyStr,
                 backend: str,
                 device: str,
                 data: ByteString,
                 batch: int = None,
                 minbatch: int = None,
                 tag: AnyStr = None,
                 inputs: Union[AnyStr, List[AnyStr]] = None,
                 outputs: Union[AnyStr, List[AnyStr]] = None) -> str:
        """
        Set the model on provided key.

        Parameters
        ----------
        key : AnyStr
            Key name
        backend : str
            Backend name. Allowed backends are TF, TORCH, TFLITE, ONNX
        device : str
            Device name. Allowed devices are CPU and GPU. If multiple GPUs are available,
            it can be specified using the format GPU:<gpu number>. For example: GPU:0
        data : bytes
            Model graph read as bytes string
        batch : int
            Number of batches for doing auto-batching
        minbatch : int
            Minimum number of samples required in a batch for model execution
        tag : AnyStr
            Any string that will be saved in RedisAI as tag for the model
        inputs : Union[AnyStr, List[AnyStr]]
            Input node(s) in the graph. Required only Tensorflow graphs
        outputs : Union[AnyStr, List[AnyStr]]
            Output node(s) in the graph Required only for Tensorflow graphs

        Returns
        -------
        str
            'OK' if success, raise an exception otherwise

        Example
        -------
        >>> # Torch model
        >>> model_path = os.path.join('path/to/TorchScriptModel.pt')
        >>> model = open(model_path, 'rb').read()
        >>> con.modelset("model", 'torch', 'cpu', model, tag='v1.0')
        'OK'
        >>> # Tensorflow model
        >>> model_path = os.path.join('/path/to/tf_frozen_graph.pb')
        >>> model = open(model_path, 'rb').read()
        >>> con.modelset('m', 'tf', 'cpu', model,
        ...              inputs=['a', 'b'], outputs=['mul'], tag='v1.0')
        'OK'
        """
        args = builder.modelset(key, backend, device, data,
                                batch, minbatch, tag, inputs, outputs)
        res = self.execute_command(*args)
        return res if not self.enable_postprocess else processor.modelset(res)