Python pymongo.errors.BulkWriteError() Examples

def load_variant_bulk(self, variants):
        """Load a bulk of variants

        Args:
            variants(iterable(scout.models.Variant))

        Returns:
            object_ids
        """
        if len(variants) == 0:
            return

        LOG.debug("Loading variant bulk")
        try:
            result = self.variant_collection.insert_many(variants)
        except (DuplicateKeyError, BulkWriteError) as err:
            # If the bulk write is wrong there are probably some variants already existing
            # In the database. So insert each variant
            for var_obj in variants:
                try:
                    self.upsert_variant(var_obj)
                except IntegrityError as err:
                    pass

        return 

def load_hpo_bulk(self, hpo_bulk):
        """Add a hpo object

        Arguments:
            hpo_bulk(list(scout.models.HpoTerm))

        Returns:
            result: pymongo bulkwrite result

        """
        LOG.debug("Loading hpo bulk")

        try:
            result = self.hpo_term_collection.insert_many(hpo_bulk)
        except (DuplicateKeyError, BulkWriteError) as err:
            raise IntegrityError(err)
        return result 

def load_hgnc_bulk(self, gene_objs):
        """Load a bulk of hgnc gene objects

        Raises IntegrityError if there are any write concerns

        Args:
            gene_objs(iterable(scout.models.hgnc_gene))

        Returns:
            result (pymongo.results.InsertManyResult)
        """

        LOG.info("Loading gene bulk with length %s", len(gene_objs))
        try:
            result = self.hgnc_collection.insert_many(gene_objs)
        except (DuplicateKeyError, BulkWriteError) as err:
            raise IntegrityError(err)

        return result 

def writeImportsTree(self, imports):
        command = {"$inc": {"count": 1}}
        bulk = self.import_coll.initialize_unordered_bulk_op()
        execute_bool = False
        for i in imports:
            dll_name = i["lib"]
            funcs = i["functions"]
            for imp_name in funcs:
                execute_bool = True
                bulk.find({"function_name": imp_name.lower(),
                           "dll_name": dll_name.lower()}).upsert().update(command)
                # print("**** Error Imports Tree ****")
                # err=str(traceback.format_exc())
                # print(err)
                # return -1
        try:
            if(execute_bool):
                bulk.execute({'w': 0})
        except BulkWriteError:
            logging.exception("MetaController(): " +
                              str("**** Error Imports Tree ****"))
            # err=str(traceback.format_exc())
            # print(err)
            return -1
        return 0 

def cache_pois(self):
        async def bulk_write(group):
            requests = []
            for item in group:
                item["_id"] = item.pop("id")
                requests.append(
                    ReplaceOne({"_id": item["_id"]}, item, upsert=True))
            try:
                await self.db.pois.bulk_write(requests)
            except BulkWriteError as e:
                self.log.exception("BWE while caching continents")

        continents = await self.call_api("continents/1/floors?ids=all")
        pois = []
        for continent in continents:
            for region in continent["regions"].values():
                for game_map in region["maps"].values():
                    for poi in game_map["points_of_interest"].values():
                        del poi["chat_link"]
                        poi["continent_id"] = continent["id"]
                        pois.append(poi)
                        if len(pois) > 200:
                            await bulk_write(pois)
                            pois = []
        print("Continents done") 

def import_database(dump, collections=None, overwrite=None):
    database = cascade.database.pymongo()

    if not isinstance(dump, list):
        raise ValueError("Dump file is not a BSON list")

    for partial_dump in dump:
        collection = partial_dump['collection']
        content = partial_dump['content']

        if collections is not None and collection not in collections:
            continue

        if len(content):
            print("Importing collection {}".format(collection))
            try:
                database[collection].insert_many(content)
            except BulkWriteError:
                replace = overwrite
                if replace is None:
                    replace = confirm("Matching documents exist in collection {}. Overwrite existing? [Y/N]".format(collection))
                for document in content:
                    doc_id = document.get('_id')

                    if doc_id is None:
                        database[collection].insert_one(document)
                    elif database[collection].find_one({'_id': doc_id}):
                        if replace:
                            database[collection].save(document)
                    else:
                        database[collection].insert_one(document) 

def save_stock_list(self, stocks):
        if len(stocks) > 0:
            try:
                stocklist_collection = self.db[self.STOCK_LIST]
                stocklist_collection.insert_many(stocks, ordered=False)
            except (DuplicateKeyError, BulkWriteError) as err:
                logger.error("save_stock_list: %i %s", len(stocks), err) 

def save_stock_historical_data(self, quote, stock_historical_data_array):
        if len(stock_historical_data_array) > 0:
            try:
                collection_name = quote + self.HISTORICAL_DATA_SUFIX
                self.create_historical_collection_if_not_exists(collection_name)
                stock_historical_data_collection = self.db[collection_name]
                stock_historical_data_collection.insert_many(stock_historical_data_array, ordered=False)
            except (DuplicateKeyError, BulkWriteError) as err:
                logger.error("save_stock_historical_data: %s %i %s", quote, len(stock_historical_data_array), err) 

def update_mongo_compound_variants(self, bulk):
        """Update the compound information for a bulk of variants in the database

            Args:
                bulk(dict): {'_id': scout.models.Variant}

        """
        requests = []
        for var_id in bulk:
            var_obj = bulk[var_id]
            if not var_obj.get("compounds"):
                continue
            # Add a request to update compounds
            operation = pymongo.UpdateOne(
                {"_id": var_obj["_id"]}, {"$set": {"compounds": var_obj["compounds"]}}
            )
            requests.append(operation)

        if not requests:
            return

        try:
            self.variant_collection.bulk_write(requests, ordered=False)
        except BulkWriteError as err:
            LOG.warning("Updating compounds failed")
            raise err 

def load_exon_bulk(self, exon_objs):
        """Load a bulk of exon objects to the database

        Arguments:
            exon_objs(iterable(scout.models.hgnc_exon))

        """
        try:
            LOG.debug("Loading exon bulk")
            result = self.exon_collection.insert_many(exon_objs)
        except (DuplicateKeyError, BulkWriteError) as err:
            raise IntegrityError(err)

        return result 

def cache_endpoint(self, endpoint, all_at_once=False):
        async def bulk_write(item_group):
            requests = []
            for item in itemgroup:
                item["_id"] = item.pop("id")
                requests.append(
                    ReplaceOne({"_id": item["_id"]}, item, upsert=True))
            try:
                await self.db[endpoint.replace("/", "_")].bulk_write(requests)
            except BulkWriteError as e:
                self.log.exception("BWE while caching {}".format(endpoint),
                                   exc_info=e)

        schema = datetime.datetime(2019, 12, 19)
        items = await self.call_api(endpoint, schema_version=schema)
        if not all_at_once:
            counter = 0
            total = len(items)
            while True:
                percentage = (counter / total) * 100
                print("Progress: {0:.1f}%".format(percentage))
                ids = ",".join(str(x) for x in items[counter:counter + 200])
                if not ids:
                    print("{} done".format(endpoint))
                    break
                itemgroup = await self.call_api(f"{endpoint}?ids={ids}",
                                                schema_version=schema)
                await bulk_write(itemgroup)
                counter += 200
        else:
            itemgroup = await self.call_api("{}?ids=all".format(endpoint),
                                            schema_version=schema)
            await bulk_write(itemgroup) 

def update(self, statement):
        from pymongo import UpdateOne
        from pymongo.errors import BulkWriteError

        data = statement.serialize()

        operations = []

        update_operation = UpdateOne(
            {'text': statement.text},
            {'$set': data},
            upsert=True
        )
        operations.append(update_operation)

        # Make sure that an entry for each response is saved
        for response_dict in data.get('in_response_to', []):
            response_text = response_dict.get('text')

            # $setOnInsert does nothing if the document is not created
            update_operation = UpdateOne(
                {'text': response_text},
                {'$set': response_dict},
                upsert=True
            )
            operations.append(update_operation)

        try:
            self.statements.bulk_write(operations, ordered=False)
        except BulkWriteError as bwe:
            # Log the details of a bulk write error
            self.logger.error(str(bwe.details))

        return statement 

def update_variant_rank(self, case_obj, variant_type="clinical", category="snv"):
        """Updates the manual rank for all variants in a case

        Add a variant rank based on the rank score
        Whenever variants are added or removed from a case we need to update the variant rank

        Args:
            case_obj(Case)
            variant_type(str)
        """
        # Get all variants sorted by rank score
        variants = self.variant_collection.find(
            {"case_id": case_obj["_id"], "category": category, "variant_type": variant_type,}
        ).sort("rank_score", pymongo.DESCENDING)

        LOG.info("Updating variant_rank for all variants")

        requests = []

        for index, var_obj in enumerate(variants):

            operation = pymongo.UpdateOne(
                {"_id": var_obj["_id"]}, {"$set": {"variant_rank": index + 1}}
            )
            requests.append(operation)

            if len(requests) < 5000:
                continue
            try:
                self.variant_collection.bulk_write(requests, ordered=False)
                requests = []
            except BulkWriteError as err:
                LOG.warning("Updating variant rank failed")
                raise err

        # Update the final bulk
        if len(requests) > 0:
            try:
                self.variant_collection.bulk_write(requests, ordered=False)
            except BulkWriteError as err:
                LOG.warning("Updating variant rank failed")
                raise err

        LOG.info("Updating variant_rank done")