Python lmdb.Environment() Examples

The following are 30 code examples of lmdb.Environment(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module lmdb , or try the search function .
Example #1
Source File: client.py    From hangar-py with Apache License 2.0 6 votes vote down vote up
def push_find_missing_schemas(self, commit, tmpDB: lmdb.Environment = None):

        if tmpDB is None:
            with tempfile.TemporaryDirectory() as tempD:
                tmpDF = os.path.join(tempD, 'test.lmdb')
                tmpDB = lmdb.open(path=tmpDF, **c.LMDB_SETTINGS)
                commiting.unpack_commit_ref(self.env.refenv, tmpDB, commit)
                c_schemaset = set(queries.RecordQuery(tmpDB).schema_hashes())
                c_schemas = list(c_schemaset)
                tmpDB.close()
        else:
            c_schemaset = set(queries.RecordQuery(tmpDB).schema_hashes())
            c_schemas = list(c_schemaset)

        request = hangar_service_pb2.FindMissingSchemasRequest()
        request.commit = commit
        request.schema_digests.extend(c_schemas)

        response = self.stub.PushFindMissingSchemas(request)
        return response 
Example #2
Source File: commiting.py    From hangar-py with Apache License 2.0 6 votes vote down vote up
def check_commit_hash_in_history(refenv, commit_hash):
    """Check if a commit hash exists in the repository history

    Parameters
    ----------
    refenv : lmdb.Environment
        refenv where the commit history is stored
    commit_hash : str
        hash of the commit to check for existence

    Returns
    -------
    bool
        True if exists, otherwise False
    """
    reftxn = TxnRegister().begin_reader_txn(refenv)
    try:
        commitParentKey = commit_parent_db_key_from_raw_key(commit_hash)
        commitParentVal = reftxn.get(commitParentKey, default=False)
        isCommitInHistory = True if commitParentVal is not False else False
    finally:
        TxnRegister().abort_reader_txn(refenv)
    return isCommitInHistory 
Example #3
Source File: heads.py    From hangar-py with Apache License 2.0 6 votes vote down vote up
def commit_hash_to_branch_name_map(branchenv: lmdb.Environment) -> dict:
    """Determine branch names which map to commit hashs

    Parameters
    ----------
    branchenv : lmdb.Environment
        db where the branch references are stored

    Returns
    -------
    dict
        keys are commit hash strings, values are list of branch names (strings)
        whose HEAD are at the key commit
    """
    outMap = defaultdict(list)
    branchNames = get_branch_names(branchenv=branchenv)
    for branchName in branchNames:
        branchHEAD = get_branch_head_commit(branchenv=branchenv, branch_name=branchName)
        outMap[branchHEAD].append(branchName)

    return outMap


# ----------------------------- Remotes --------------------------------------- 
Example #4
Source File: commiting.py    From hangar-py with Apache License 2.0 6 votes vote down vote up
def tmp_cmt_env(refenv: lmdb.Environment, commit_hash: str):
    """create temporary unpacked lmdb environment from compressed structure

    Parameters
    ----------
    refenv : lmdb.Environment
        lmdb environment where the commit refs are stored
    commit_hash : str
        hash of the commit to get the contents of

    Returns
    -------
    lmdb.Environment
        environment with all db contents from ``commit`` unpacked
    """
    with tempfile.TemporaryDirectory() as tmpdir:
        tmpDF = os.path.join(tmpdir, 'test.lmdb')
        with closing(
                lmdb.open(tmpDF, sync=False, writemap=True, **LMDB_SETTINGS)
        ) as tmpDB:
            unpack_commit_ref(refenv, tmpDB, commit_hash)
            yield tmpDB 
Example #5
Source File: commiting.py    From hangar-py with Apache License 2.0 6 votes vote down vote up
def _commit_ref(stageenv: lmdb.Environment) -> DigestAndBytes:
    """Query and format all staged data records, and format it for ref storage.

    Parameters
    ----------
    stageenv : lmdb.Environment
        lmdb environment where the staged record data is actually stored.

    Returns
    -------
    DigestAndBytes
        Serialized and compressed version of all staged record data along with
        digest of commit refs.
    """
    from .queries import RecordQuery  # needed to avoid cyclic import

    querys = RecordQuery(dataenv=stageenv)
    allRecords = tuple(querys._traverse_all_records())
    res = commit_ref_db_val_from_raw_val(allRecords)
    return res


# -------------------- Format ref k/v pairs and write the commit to disk ---------------- 
Example #6
Source File: diff.py    From hangar-py with Apache License 2.0 6 votes vote down vote up
def _diff3(a_env: lmdb.Environment,
               m_env: lmdb.Environment,
               d_env: lmdb.Environment) -> DiffAndConflictsDB:
        """Three way diff and conflict finder from ancestor, master, and dev commits.

        Parameters
        ----------
        a_env : lmdb.Environment
            unpacked lmdb environment for the ancestor commit
        m_env : lmdb.Environment
            unpacked lmdb environment for the master commit, current HEAD
        d_env : lmdb.Environment
            unpacked lmdb environment for the dev commit, compare to HEAD

        Returns
        -------
        DiffAndConflictsDB
            structure containing (`additions`, `deletions`, `mutations`) for
            diff, as well as the ConflictRecord struct.
        """
        it = ((a_env, m_env), (a_env, d_env), (d_env, m_env))
        diffs = tuple(starmap(diff_envs, it))  # significant perf improvement by map.
        conflict = find_conflicts(diffs[0], diffs[1])
        return DiffAndConflictsDB(diff=diffs[2], conflict=conflict) 
Example #7
Source File: txnctx.py    From hangar-py with Apache License 2.0 6 votes vote down vote up
def begin_reader_txn(self, lmdbenv: lmdb.Environment,
                         buffer: bool = False) -> lmdb.Transaction:
        """Start a reader only txn for the given environment

        If there a read-only transaction for the same environment already exists
        then the same reader txn handle will be returned, and will not close
        until all operations on that handle have said they are finished.

        Parameters
        ----------
        lmdbenv : lmdb.Environment
            the environment to start the transaction in.
        buffer : bool, optional
            weather a buffer transaction should be used (the default is False,
            which means no buffers are returned)

        Returns
        -------
        lmdb.Transaction
            handle to the lmdb transaction.
        """
        if self.ReaderAncestors[lmdbenv] == 0:
            self.ReaderTxn[lmdbenv] = lmdbenv.begin(write=False, buffers=buffer)
        self.ReaderAncestors[lmdbenv] += 1
        return self.ReaderTxn[lmdbenv] 
Example #8
Source File: txnctx.py    From hangar-py with Apache License 2.0 6 votes vote down vote up
def begin_writer_txn(self, lmdbenv: lmdb.Environment,
                         buffer: bool = False) -> lmdb.Transaction:
        """Start a write enabled transaction on the given environment

        If multiple write transactions are requested for the same handle, only
        one instance of the transaction handle will be returened, and will not
        close until all operations on that handle have requested to close

        Parameters
        ----------
        lmdbenv : lmdb.Environment
            the environment to open the transaction on
        buffer : bool, optional
            if buffer objects should be used (the default is False, which does
            not use buffers)

        Returns
        -------
        lmdb.Transaction
            transaction handle to perform operations on
        """
        if self.WriterAncestors[lmdbenv] == 0:
            self.WriterTxn[lmdbenv] = lmdbenv.begin(write=True, buffers=buffer)
        self.WriterAncestors[lmdbenv] += 1
        return self.WriterTxn[lmdbenv] 
Example #9
Source File: hashs.py    From hangar-py with Apache License 2.0 6 votes vote down vote up
def remove_stage_hash_records_from_hashenv(hashenv, stagehashenv):
    """Remove references to data additions during a hard reset

    For every hash record in stagehashenv, remove the corresponding k/v pair
    from the hashenv db. This is a dangerous operation if the stagehashenv was
    not appropriately constructed!!!

    Parameters
    ----------
    hashenv : lmdb.Environment
        db where all the permanent hash records are stored
    stagehashenv : lmdb.Environment
        db where all the staged hash records to be removed are stored.
    """
    stageHashKeys = HashQuery(stagehashenv).gen_all_hash_keys_db()
    hashtxn = TxnRegister().begin_writer_txn(hashenv)
    for hashKey in stageHashKeys:
        hashtxn.delete(hashKey)
    TxnRegister().commit_writer_txn(hashenv) 
Example #10
Source File: heads.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def get_branch_head_commit(branchenv, branch_name):
    """Find the commit hash which corresponds to the HEAD of a particular branch.

    Parameters
    ----------
    branchenv: lmdb.Environment
        lmdb environment for the branch spec
    branch_name: str
        name of the branch to find the head commit hash for

    Returns
    -------
    str
        the commit hash of the branch head

    Raises
    ------
    ValueError
        if `branch_name` does not exist in the repository
    """
    requestedBranchKey = repo_branch_head_db_key_from_raw_key(branch_name)
    branchtxn = TxnRegister().begin_reader_txn(branchenv)
    try:
        branchNameVal = branchtxn.get(requestedBranchKey, default=False)
        if branchNameVal is False:
            err = f'branch with name: {branch_name} does not exist. cannot get head.'
            raise ValueError(err)
    finally:
        TxnRegister().abort_reader_txn(branchenv)

    commit_hash = repo_branch_head_raw_val_from_db_val(branchNameVal)
    return commit_hash 
Example #11
Source File: heads.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def set_branch_head_commit(branchenv, branch_name, commit_hash):
    """Update an existing branch HEAD to point to a new commit hash.

    Does not update stage or refenv contents. If the current HEAD of the branch
    == the new commit hash, no operation will occur and an exception will be
    thrown.

    Parameters
    ----------
    branchenv : lmdb.Environment
        lmdb environment where the branch records are kept
    branch_name : string
        Name of the branch to update the HEAD commit of
    commit_hash : string
        Commit hash to update the branch HEAD to point to.

    Returns
    -------
    string
        Commit hash of the new branch head if the operation was successful.

    Raises
    ------
    ValueError
        If the current HEAD is the same as the new commit hash.
    """
    currentHeadCommit = get_branch_head_commit(branchenv=branchenv, branch_name=branch_name)
    if currentHeadCommit == commit_hash:
        err = f'Current branch: {branch_name} HEAD: {currentHeadCommit} is same as the '\
              f'requested updated HEAD: {commit_hash}, no-op performed'
        raise ValueError(err)

    branchtxn = TxnRegister().begin_writer_txn(branchenv)
    try:
        branchHeadKey = repo_branch_head_db_key_from_raw_key(branch_name)
        branchHeadVal = repo_branch_head_db_val_from_raw_val(commit_hash)
        branchtxn.put(branchHeadKey, branchHeadVal)
    finally:
        TxnRegister().commit_writer_txn(branchenv)

    return commit_hash 
Example #12
Source File: integrity.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def _verify_column_integrity(hashenv: lmdb.Environment, repo_path: Path):

    hq = hashs.HashQuery(hashenv)
    narrays, nremote = hq.num_data_records(), 0
    array_kvs = hq.gen_all_data_digests_and_parsed_backend_specs()
    try:
        bes = {}
        for digest, spec in tqdm(array_kvs, total=narrays, desc='verifying column data'):
            if spec.backend not in bes:
                bes[spec.backend] = BACKEND_ACCESSOR_MAP[spec.backend](repo_path, None, None)
                bes[spec.backend].open(mode='r')
            if spec.islocal is False:
                nremote += 1
                continue
            data = bes[spec.backend].read_data(spec)
            tcode = hashmachine.hash_type_code_from_digest(digest)

            hash_func = hashmachine.hash_func_from_tcode(tcode)
            calc_digest = hash_func(data)
            if calc_digest != digest:
                raise RuntimeError(
                    f'Data corruption detected for array. Expected digest `{digest}` '
                    f'currently mapped to spec `{spec}`. Found digest `{calc_digest}`')
        if nremote > 0:
            warnings.warn(
                'Can not verify integrity of partially fetched array data references. '
                f'For complete proof, fetch all remote data locally. Did not verify '
                f'{nremote}/{narrays} arrays', RuntimeWarning)
    finally:
        for be in bes.keys():
            bes[be].close() 
Example #13
Source File: heads.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def get_branch_names(branchenv):
    """get a list of all branches in the repository.

    Parameters
    ----------
    branchenv : lmdb.Environment
        lmdb environment storing the branch records.

    Returns
    -------
    list of str
        list of branch names active in the repository.
    """
    branchStartKey = K_BRANCH.encode()  # TODO: This is odd, why??
    branchNames = []
    branchTxn = TxnRegister().begin_reader_txn(branchenv)
    try:
        with branchTxn.cursor() as cursor:
            cursor.first()
            branchRangeExists = cursor.set_range(branchStartKey)
            while branchRangeExists:
                branchKey = cursor.key()
                if branchKey.startswith(branchStartKey):
                    name = repo_branch_head_raw_key_from_db_key(branchKey)
                    branchNames.append(name)
                    branchRangeExists = cursor.next()
                else:
                    branchRangeExists = False
        cursor.close()
    finally:
        TxnRegister().abort_reader_txn(branchenv)

    return branchNames 
Example #14
Source File: heads.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def writer_lock_held(branchenv):
    """Check to see if the writer lock is free before attempting to acquire it.

    Parameters
    ----------
    branchenv : lmdb.Environment
        lmdb environment where the writer lock is stored

    Returns
    -------
    bool
        True if the lock is available to take, False if it is currently held.
    """
    writerLockKey = repo_writer_lock_db_key()
    writerLockSentinalVal = repo_writer_lock_sentinal_db_val()
    branchtxn = TxnRegister().begin_reader_txn(branchenv)
    try:
        currentWriterLockVal = branchtxn.get(writerLockKey)
        if currentWriterLockVal == writerLockSentinalVal:
            lockAvailable = True
        elif currentWriterLockVal is None:
            # on first initialization, writer lock key/val is not set.
            lockAvailable = True
        else:
            lockAvailable = False
    finally:
        TxnRegister().abort_reader_txn(branchenv)
    return lockAvailable 
Example #15
Source File: heads.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def get_remote_address(branchenv: lmdb.Environment, name: str) -> str:
    """Retrieve the IO:PORT of the remote server for a given name

    Parameters
    ----------
    branchenv : lmdb.Environment
        db where the branch (and remote) references are stored
    name : str
        name of the remote to fetch

    Raises
    ------
    KeyError
        if a remote with the provided name does not exist

    Returns
    -------
    str
        IP:PORT of the recorded remote server.
    """
    dbKey = remote_db_key_from_raw_key(name)
    branchTxn = TxnRegister().begin_reader_txn(branchenv)
    try:
        dbVal = branchTxn.get(dbKey, default=False)
    finally:
        TxnRegister().abort_reader_txn(branchenv)

    if dbVal is False:
        msg = f'No remote with the name: {name} exists in the repo.'
        raise KeyError(msg)
    else:
        remote_address = remote_raw_val_from_db_val(dbVal)
        return remote_address 
Example #16
Source File: heads.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def remove_remote(branchenv: lmdb.Environment, name: str) -> str:
    """remove a remote reference with the provided name.

    Parameters
    ----------
    branchenv : lmdb.Environment
        db where the branch (and remote) records are stored.
    name : str
        name of the remote to remove from the repo

    Raises
    ------
    ValueError
        if a remote with the provided name does not exist

    Returns
    -------
    str
        IP:PORT of the remote with provided name (which was removed)
    """
    dbKey = remote_db_key_from_raw_key(name)
    branchTxn = TxnRegister().begin_writer_txn(branchenv)
    try:
        dbVal = branchTxn.pop(dbKey)
    finally:
        TxnRegister().commit_writer_txn(branchenv)

    if dbVal is None:
        msg = f'No remote with the name: {name} exists in the repo.'
        raise ValueError(msg)

    remote_address = remote_raw_val_from_db_val(dbVal)
    return remote_address 
Example #17
Source File: heads.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def get_remote_names(branchenv):
    """get a list of all remotes in the repository.

    Parameters
    ----------
    branchenv : lmdb.Environment
        lmdb environment storing the branch records.

    Returns
    -------
    list of str
        list of remote names active in the repository.
    """
    remoteStartKey = K_REMOTES.encode()
    remoteNames = []
    branchTxn = TxnRegister().begin_reader_txn(branchenv)
    try:
        with branchTxn.cursor() as cursor:
            cursor.first()
            remoteRangeExists = cursor.set_range(remoteStartKey)
            while remoteRangeExists:
                remoteKey = cursor.key()
                if remoteKey.startswith(remoteStartKey):
                    name = remote_raw_key_from_db_key(remoteKey)
                    remoteNames.append(name)
                    remoteRangeExists = cursor.next()
                else:
                    remoteRangeExists = False
        cursor.close()
    finally:
        TxnRegister().abort_reader_txn(branchenv)

    return remoteNames 
Example #18
Source File: queries.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def __init__(self, dataenv: lmdb.Environment):
        self._dataenv = dataenv

# ------------------ traversing the unpacked records ------------------------------------ 
Example #19
Source File: lmdb_writer.py    From sanet_relocal_demo with GNU General Public License v3.0 5 votes vote down vote up
def __start_session__(self):
        self.__lmdb_env__ = lmdb.Environment(self.lmdb_path, map_size=1099511627776)
        self.__lmdb_txn__ = self.__lmdb_env__.begin(write=True) 
Example #20
Source File: integrity.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def _verify_schema_integrity(hashenv: lmdb.Environment):

    hq = hashs.HashQuery(hashenv)
    schema_kvs = hq.gen_all_schema_digests_and_parsed_specs()
    nschemas = hq.num_schema_records()
    for digest, val in tqdm(schema_kvs, total=nschemas, desc='verifying schemas'):
        tcode = hashmachine.hash_type_code_from_digest(digest)
        hash_func = hashmachine.hash_func_from_tcode(tcode)
        calc_digest = hash_func(val)
        if calc_digest != digest:
            raise RuntimeError(
                f'Data corruption detected for schema. Expected digest `{digest}` '
                f'currently mapped to spec `{val}`. Found digest `{calc_digest}`') 
Example #21
Source File: integrity.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def _verify_commit_tree_integrity(refenv: lmdb.Environment):

    initialCmt = None
    all_commits = set(commiting.list_all_commits(refenv))
    reftxn = TxnRegister().begin_reader_txn(refenv)
    try:
        for cmt in tqdm(all_commits, desc='verifying commit trees'):
            pKey = parsing.commit_parent_db_key_from_raw_key(cmt)
            pVal = reftxn.get(pKey, default=False)
            if pVal is False:
                raise RuntimeError(
                    f'Data corruption detected for parent ref of commit `{cmt}`. '
                    f'Parent ref not recorded in refs db.')

            p_val = parsing.commit_parent_raw_val_from_db_val(pVal)
            parents = p_val.ancestor_spec
            if parents.master_ancestor != '':
                if parents.master_ancestor not in all_commits:
                    raise RuntimeError(
                        f'Data corruption detected in commit tree. Commit `{cmt}` '
                        f'with ancestors val `{parents}` references non-existing '
                        f'master ancestor `{parents.master_ancestor}`.')
            if parents.dev_ancestor != '':
                if parents.dev_ancestor not in all_commits:
                    raise RuntimeError(
                        f'Data corruption detected in commit tree. Commit `{cmt}` '
                        f'with ancestors val `{parents}` references non-existing '
                        f'dev ancestor `{parents.dev_ancestor}`.')
            if (parents.master_ancestor == '') and (parents.dev_ancestor == ''):
                if initialCmt is not None:
                    raise RuntimeError(
                        f'Commit tree integrity compromised. Multiple "initial" (commits '
                        f'with no parents) found. First `{initialCmt}`, second `{cmt}`')
                else:
                    initialCmt = cmt
    finally:
        TxnRegister().abort_reader_txn(refenv) 
Example #22
Source File: integrity.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def _verify_commit_ref_digests_exist(hashenv: lmdb.Environment, refenv: lmdb.Environment):

    all_commits = commiting.list_all_commits(refenv)
    datatxn = TxnRegister().begin_reader_txn(hashenv, buffer=True)
    try:
        with datatxn.cursor() as cur:
            for cmt in tqdm(all_commits, desc='verifying commit ref digests'):
                with commiting.tmp_cmt_env(refenv, cmt) as tmpDB:
                    rq = queries.RecordQuery(tmpDB)
                    array_data_digests = set(rq.data_hashes())
                    schema_digests = set(rq.schema_hashes())

                    for datadigest in array_data_digests:
                        dbk = hash_data_db_key_from_raw_key(datadigest)
                        exists = cur.set_key(dbk)
                        if exists is False:
                            raise RuntimeError(
                                f'Data corruption detected in commit refs. Commit `{cmt}` '
                                f'references array data digest `{datadigest}` which does not '
                                f'exist in data hash db.')

                    for schemadigest in schema_digests:
                        dbk = hash_schema_db_key_from_raw_key(schemadigest)
                        exists = cur.set_key(dbk)
                        if exists is False:
                            raise RuntimeError(
                                f'Data corruption detected in commit refs. Commit `{cmt}` '
                                f'references schema digest `{schemadigest}` which does not '
                                f'exist in data hash db.')

    finally:
        TxnRegister().abort_reader_txn(hashenv) 
Example #23
Source File: integrity.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def run_verification(branchenv: lmdb.Environment,
                     hashenv: lmdb.Environment,
                     refenv: lmdb.Environment,
                     repo_path: Path):

    _verify_branch_integrity(branchenv, refenv)
    _verify_commit_tree_integrity(refenv)
    _verify_commit_ref_digests_exist(hashenv, refenv)
    _verify_schema_integrity(hashenv)
    _verify_column_integrity(hashenv, repo_path) 
Example #24
Source File: client.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def push_find_missing_hash_records(self, commit, tmpDB: lmdb.Environment = None):

        if tmpDB is None:
            with tempfile.TemporaryDirectory() as tempD:
                tmpDF = os.path.join(tempD, 'test.lmdb')
                tmpDB = lmdb.open(path=tmpDF, **c.LMDB_SETTINGS)
                commiting.unpack_commit_ref(self.env.refenv, tmpDB, commit)
                c_hashs_schemas = queries.RecordQuery(tmpDB).data_hash_to_schema_hash()
                c_hashes = list(set(c_hashs_schemas.keys()))
                tmpDB.close()
        else:
            c_hashs_schemas = queries.RecordQuery(tmpDB).data_hash_to_schema_hash()
            c_hashes = list(set(c_hashs_schemas.keys()))

        c_hashs_raw = [chunks.serialize_ident(digest, '') for digest in c_hashes]
        raw_pack = chunks.serialize_record_pack(c_hashs_raw)
        pb2_func = hangar_service_pb2.FindMissingHashRecordsRequest
        cIter = chunks.missingHashRequestIterator(commit, raw_pack, pb2_func)

        responses = self.stub.PushFindMissingHashRecords(cIter)
        for idx, response in enumerate(responses):
            if idx == 0:
                hBytes, offset = bytearray(response.total_byte_size), 0
            size = len(response.hashs)
            hBytes[offset: offset + size] = response.hashs
            offset += size

        uncompBytes = blosc.decompress(hBytes)
        s_missing_raw = chunks.deserialize_record_pack(uncompBytes)
        s_mis_hsh = [chunks.deserialize_ident(raw).digest for raw in s_missing_raw]
        s_mis_hsh_sch = [(s_hsh, c_hashs_schemas[s_hsh]) for s_hsh in s_mis_hsh]
        return s_mis_hsh_sch 
Example #25
Source File: diff.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def __init__(self, branchenv: lmdb.Environment, refenv: lmdb.Environment, *args, **kwargs):

        self._branchenv: lmdb.Environment = branchenv
        self._refenv: lmdb.Environment = refenv 
Example #26
Source File: diff.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def __init__(self, stageenv: lmdb.Environment, branch_name: str, *args, **kwargs):

        super().__init__(*args, **kwargs)
        self._stageenv: lmdb.Environment = stageenv
        self._branch_name: str = branch_name 
Example #27
Source File: engine.py    From eavatar-me with Apache License 2.0 5 votes vote down vote up
def start(self, ctx=None):
        logger.debug("Starting data engine...")

        # register with the context
        if ctx:
            ctx.bind('dataengine', self)
        if not self.datapath:
            self.datapath = os.path.join(environ.data_dir(), 'stores')
            if not os.path.exists(self.datapath):
                os.makedirs(self.datapath)
        logger.debug("Data path: %s", self.datapath)

        try:
            self.database = lmdb.Environment(self.datapath,
                                             map_size=2000000000,
                                             max_dbs=1024)
            with self.database.begin(write=False) as txn:
                cur = txn.cursor()
                for k, v in iter(cur):
                    logger.debug("Found existing store: %s", k)
                    _db = self.database.open_db(k, create=False)
                    self.stores[k] = Store(k, _db, self)
        except lmdb.Error:
            logger.exception("Failed to open database.", exc_info=True)
            raise

        logger.debug("Data engine started.") 
Example #28
Source File: lmdb_nolock_database.py    From sawtooth-core with Apache License 2.0 5 votes vote down vote up
def __init__(self, filename, flag):
        """Constructor for the LMDBNoLockDatabase class.

        Args:
            filename (str): The filename of the database file.
            flag (str): a flag indicating the mode for opening the database.
                Refer to the documentation for anydbm.open().
        """
        super(LMDBNoLockDatabase, self).__init__()

        create = bool(flag == 'c')

        if flag == 'n':
            if os.path.isfile(filename):
                os.remove(filename)
            create = True

        self._lmdb = lmdb.Environment(
            path=filename,
            map_size=1024**4,
            map_async=True,
            writemap=True,
            readahead=False,
            subdir=False,
            create=create,
            lock=True)

    # pylint: disable=no-value-for-parameter 
Example #29
Source File: txnctx.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def __init__(self):
        self.WriterAncestors = Counter()
        self.ReaderAncestors = Counter()
        self.WriterTxn: MutableMapping[lmdb.Environment, lmdb.Transaction] = {}
        self.ReaderTxn: MutableMapping[lmdb.Environment, lmdb.Transaction] = {} 
Example #30
Source File: txnctx.py    From hangar-py with Apache License 2.0 5 votes vote down vote up
def commit_writer_txn(self, lmdbenv: lmdb.Environment) -> bool:
        """Commit changes made in a write-enable transaction handle

        As multiple objects can have references to the same open transaction handle,
        the data is not actually committed until all open transactions have called
        the commit method.

        Parameters
        ----------
        lmdbenv : lmdb.Environment
            the environment handle used to open the transaction

        Raises
        ------
        RuntimeError
            If the internal reference counting gets out of sync

        Returns
        -------
        bool
            True if this operation actually committed, otherwise false
            if other objects have references to the same (open) handle
        """
        ancestors = self.WriterAncestors[lmdbenv]
        if ancestors == 0:
            msg = f'hash ancestors are zero but commit called on {lmdbenv}'
            raise RuntimeError(msg)
        elif ancestors == 1:
            self.WriterTxn[lmdbenv].commit()
            self.WriterTxn.__delitem__(lmdbenv)
            ret = True
        else:
            ret = False
        self.WriterAncestors[lmdbenv] -= 1
        return ret