Python pymongo.ASCENDING Examples

def _get_collection(self, collection_name=None):
        if collection_name:
            if collection_name in self.allowed_collections:
                coins = self.client.ratesx[collection_name]
            else:
                raise KeyError("Not allowed collection name: %s" % collection_name)
        else:
            coins = self.coins

        if collection_name \
            and collection_name not in self.client.ratesx.collection_names():

            if collection_name.startswith('coins_'):
                coins.create_index([('symbol', ASCENDING)], unique=False)
                coins.create_index([('timestamp', ASCENDING)], unique=False)
                coins.create_index([('symbol', ASCENDING),
                                    ('timestamp', ASCENDING)], unique=True)

            if collection_name.startswith('currencies_'):
                coins.create_index([('timestamp', ASCENDING)], unique=False)

        return coins 

def save_account(message, collection=DATABASE.account):
    """save account

    Arguments:
        message {[type]} -- [description]

    Keyword Arguments:
        collection {[type]} -- [description] (default: {DATABASE})
    """
    try:
        collection.create_index(
            [("account_cookie", ASCENDING), ("user_cookie", ASCENDING), ("portfolio_cookie", ASCENDING)], unique=True)
    except:
        pass
    collection.update(
        {'account_cookie': message['account_cookie'], 'portfolio_cookie':
            message['portfolio_cookie'], 'user_cookie': message['user_cookie']},
        {'$set': message},
        upsert=True
    ) 

def save_position(message, collection=DATABASE.positions):
    """save account

    Arguments:
        message {[type]} -- [description]

    Keyword Arguments:
        collection {[type]} -- [description] (default: {DATABASE})
    """
    try:
        collection.create_index(
            [("account_cookie", ASCENDING), ("portfolio_cookie", ASCENDING), ("user_cookie", ASCENDING), ("position_id", ASCENDING)], unique=True)
    except:
        pass
    collection.update(
        {'account_cookie': message['account_cookie'], 'position_id': message['position_id'],
            'portfolio_cookie': message['portfolio_cookie'], 'user_cookie': message['user_cookie']},
        {'$set': message},
        upsert=True
    ) 

def updateSectorStockCodes(self, sectorCode, date, codes):
        collection = self._client[self.sectorCodeDbMap[sectorCode]][date]

        # create index
        collection.create_index([('code', pymongo.ASCENDING)], unique=True)

        # update into DB
        try:
            for code in codes:
                flt = {'code': code['code']}
                collection.update_one(flt, {'$set': {'name': code['name']}}, upsert=True)

        except Exception as ex:
            self._info.print("更新[{0}]股票代码数据[{1}]到MongoDB异常:{2}".format(DyStockCommon.sectors[sectorCode], date, str(ex) + ', ' + str(ex.details)), DyLogData.error)
            return False

        return True 

def getStockMarketDate(self, code, name=None):
        """
            获取个股上市日期
            由于数据库的数据限制，有可能是个股数据在数据库里的最早信息
        """
        collection = self._getStockDaysDb()[code]

        flt = {'datetime': {'$lt':datetime.now()}}

        try:
            cursor = collection.find(flt).sort('datetime', pymongo.ASCENDING).limit(1)
        except Exception as ex:
            self._info.print("MongoDB Exception({0}): @getStockMarketDate{1}:{2}, 日线数据".format(str(ex) + ', ' + str(ex.details),
                                                                                                    code, name),
                             DyLogData.error)
            return None

        for d in cursor:
            return d['datetime'].strftime('%Y-%m-%d')
        
        return None 

def updateStockCodes(self, codes):
        collection = self._getCodeTableCollection()

        # create index
        collection.create_index([('code', pymongo.ASCENDING)], unique=True)

        # update into DB
        try:
            for code in codes:
                flt = {'code': code['code']}
                collection.update_one(flt, {'$set':{'name': code['name']}}, upsert=True)

        except Exception as ex:
            self._info.print("更新股票代码数据到MongoDB异常:{0}".format(str(ex) + ', ' + str(ex.details)), DyLogData.error)
            return False

        return True 

def list(self, request):
        await require(request, Permissions.view)
        possible_fields = [k.name for k in self._schema.keys]
        q = validate_query(request.query, possible_fields)
        paging = calc_pagination(q, self._primary_key)

        filters = q.get('_filters')
        query = {}
        if filters:
            query = create_filter(filters, self._schema)

        sort_direction = ASCENDING if paging.sort_dir == ASC else DESCENDING

        cursor = (self._collection.find(query)
                  .skip(paging.offset)
                  .limit(paging.limit)
                  .sort(paging.sort_field, sort_direction))

        entities = await cursor.to_list(paging.limit)
        count = await self._collection.count_documents(query)
        headers = {'X-Total-Count': str(count)}
        return json_response(entities, headers=headers) 

def updateTradeDays(self, dates):
        collection = self._getTradeDayTableCollection()

        # create index
        collection.create_index([('datetime', pymongo.ASCENDING)], unique=True)

        # update into DB
        try:
            for date in dates:
                flt = {'datetime': date['datetime']}
                result = collection.update_one(flt, {'$set':{'tradeDay': date['tradeDay']}}, upsert=True)
                if not (result.acknowledged and (result.matched_count == 1 or result.upserted_id is not None)):
                    self._info.print("更新交易日数据到MongoDB失败: date={}, raw_result={}".format(date, result.raw_result), DyLogData.error)
                    return False
        except Exception as ex:
            self._info.print("更新交易日数据到MongoDB异常: {}".format(str(ex) + ', ' + str(ex.details)), DyLogData.error)
            return False

        return True 

def test_rename(chunkstore_lib):
    df = create_test_data(size=10, cols=5)

    chunkstore_lib.write('test', df, chunk_size='D')
    assert_frame_equal(chunkstore_lib.read('test'), df)
    chunkstore_lib.rename('test', 'new_name')
    assert_frame_equal(chunkstore_lib.read('new_name'), df)

    with pytest.raises(Exception) as e:
        chunkstore_lib.rename('new_name', 'new_name')
    assert('already exists' in str(e.value))

    with pytest.raises(NoDataFoundException) as e:
        chunkstore_lib.rename('doesnt_exist', 'temp')
    assert('No data found for doesnt_exist' in str(e.value))

    assert('test' not in chunkstore_lib.list_symbols())

    # read out all chunks that have symbol set to 'test'. List should be empty
    chunks = []
    for x in chunkstore_lib._collection.find({SYMBOL: 'test'}, sort=[(START, pymongo.ASCENDING)],):
        chunks.append(x)

    assert(len(chunks) == 0) 

def _ensure_index(collection):
        try:
            collection.create_index([('symbol', pymongo.HASHED)], background=True)
            # We keep it only for its uniqueness
            collection.create_index([('symbol', pymongo.ASCENDING),
                                     ('sha', pymongo.ASCENDING)], unique=True, background=True)
            # TODO: When/if we remove the segments->versions pointers implementation and keep only the forward pointers,
            #       we can remove the 'parent' from the index.
            collection.create_index([('symbol', pymongo.ASCENDING),
                                     ('parent', pymongo.ASCENDING),
                                     ('segment', pymongo.ASCENDING)], unique=True, background=True)
            # Used for efficient SHA-based read queries that have index ranges
            collection.create_index([('symbol', pymongo.ASCENDING),
                                     ('sha', pymongo.ASCENDING),
                                     ('segment', pymongo.ASCENDING)], unique=True, background=True)
        except OperationFailure as e:
            if "can't use unique indexes" in str(e):
                return
            raise 

def get_block_indexes_for_dates(start_dt=None, end_dt=None):
    """Returns a 2 tuple (start_block, end_block) result for the block range that encompasses the given start_date
    and end_date unix timestamps"""
    if start_dt is None:
        start_block_index = config.BLOCK_FIRST
    else:
        start_block = config.mongo_db.processed_blocks.find_one({"block_time": {"$lte": start_dt}}, sort=[("block_time", pymongo.DESCENDING)])
        start_block_index = config.BLOCK_FIRST if not start_block else start_block['block_index']

    if end_dt is None:
        end_block_index = config.state['my_latest_block']['block_index']
    else:
        end_block = config.mongo_db.processed_blocks.find_one({"block_time": {"$gte": end_dt}}, sort=[("block_time", pymongo.ASCENDING)])
        if not end_block:
            end_block_index = config.mongo_db.processed_blocks.find_one(sort=[("block_index", pymongo.DESCENDING)])['block_index']
        else:
            end_block_index = end_block['block_index']
    return (start_block_index, end_block_index) 

def extract(self): # TODO : Should be an exporter plugin
        graph = {
            'meta': {},  # self.__meta,
            'properties': {}  # self.__properties
        }

        graph['nodes'] = list()
        for v in self.__vertices.find().sort('id', pymongo.ASCENDING):
            v.pop("_id")  # Remove MongoDB document ID
            graph['nodes'].append(v)

        graph['edges'] = list()
        for e in self.__edges.find().sort("src", pymongo.ASCENDING):
            e.pop("_id")  # Remove MongoDB document ID
            graph['edges'].append(e)

        graph['tokens'] = list();
        for t in self.__tokens.find().sort('id', pymongo.ASCENDING):
            t.pop("_id")  # Remove MongoDB document ID
            t['id'] = str(t['id'])
            t['ts'] = time.mktime(t['ts'].timetuple())
            graph['tokens'].append(t)

        return graph 

def get_data(self, symbol, start, end, symbol_in_column=True):
        """
        Returns a dataframe of the symbol data requested.
        """
        from pymongo import ASCENDING
        symbol = str(symbol).upper()
        results = self.database[symbol].find({'_id': \
                              {'$gte': start, '$lte': end}}\
                              ).sort('datetime', ASCENDING)
        ret = pd.DataFrame.from_dict(list(results))
        if len(ret) < 1:
            raise NoDataException()
        ret.rename(columns={'open': 'Open', \
                            'high': 'High', \
                            'low': 'Low', \
                            'close': 'Close', \
                            'volume': 'Volume', \
                            'adj_close': 'Adj Close', \
                            '_id': 'Date'}, \
                           inplace=True)
        ret = ret.set_index('Date')
        if symbol_in_column:
            ret.rename(columns=lambda name: '%s_%s' %(symbol, name), inplace=True)
        return ret 

def _index_list(key_or_list, direction=None):
    """Helper to generate a list of (key, direction) pairs.

    Takes such a list, or a single key, or a single key and direction.
    """
    if direction is not None:
        return [(key_or_list, direction)]
    else:
        if isinstance(key_or_list, string_type):
            return [(key_or_list, ASCENDING)]
        elif not isinstance(key_or_list, (list, tuple)):
            raise TypeError("if no direction is specified, "
                            "key_or_list must be an instance of list")
        return key_or_list 

def nextUnprocessed(self) -> Opt[IrwinQueue]:
        irwinQueueBSON = self.irwinQueueColl.find_one_and_delete(
            filter={},
            sort=[("date", pymongo.ASCENDING)])
        return None if irwinQueueBSON is None else IrwinQueueBSONHandler.reads(irwinQueueBSON) 

def oldestNonEngine(self) -> Opt[Player]:
        playerBSON = self.playerColl.find_one_and_update(
            filter={'$or': [{'engine': False}, {'engine': None}], 'date': {'$lt': datetime.now() - timedelta(days=30)}},
            update={'$set': {'date': datetime.now()}},
            sort=[('date', pymongo.ASCENDING)])
        return None if playerBSON is None else PlayerBSONHandler.reads(playerBSON) 

def _query_collection(self, collection, query, projection, **kwargs):

        if projection:
            iterator = collection.find(query, projection)
        else:
            iterator = collection.find(query)

        try:
            sort_key, sort_order = kwargs['sort']
            iterator = iterator.sort(sort_key, pymongo.ASCENDING if sort_order > 0 else pymongo.DESCENDING)
        except:
            pass

        try:
            offset = int(kwargs['offset'])
            iterator = iterator.skip(offset)
        except:
            pass

        try:
            limit = int(kwargs['limit'])
            iterator = iterator.limit(limit)
        except:
            pass

        for element in iterator:
            del element["_id"]
            yield element 

def update_index(self, **kwargs):
        assert "key" in kwargs, "update_index: key not defined!"
        assert "type" in kwargs, "update_index: type not defined!"
        assert kwargs['type'] in ['vertex', 'edge', 'token'], "update_index: type needs to be vertex, edge or token!"

        if kwargs["type"] == "vertex":
            self.__vertices.create_index([ (kwargs['key'], pymongo.ASCENDING) ])
        elif kwargs["type"] == "edge":
            self.__edges.create_index([ (kwargs['key'], pymongo.ASCENDING) ])
        else:
            self.__tokens.create_index([ (kwargs['key'], pymongo.ASCENDING) ])

        return (kwargs['type'], kwargs['key']) 

def test_indexes(db):
    B = 'test_indexes'

    # index a key, 'a'. we should see that index when listing indexes.
    db.ensure_index(B, {'a': pymongo.ASCENDING})
    indexes = list(db._bucket(B).list_indexes())
    assert any([i.get('key').get('a') is not None for i in indexes])

    # drop indexes. we shouldn't see an index on 'a' now.
    db.drop_all_indexes(B)
    indexes = list(db._bucket(B).list_indexes())
    assert all([i.get('key').get('a') is None for i in indexes]) 

def get_patches(self):
        """
              Get patch from DB

              Args: None

              Return: first_file_id, cursor(with patch files without first init file)
              """
        cursor = self.collection.find(
            {
                "marker": self.marker_db_init}
        ).sort("received", pymongo.DESCENDING).limit(1)

        row = cursor.next()
        date = row["received"]
        first_file_id = row["_id"]

        cursor = self.collection.find(
            {
                "marker": {"$gte": self.marker_db_init},
                "received": {"$gte": date}
            }
        ).sort("received", pymongo.ASCENDING)

        LOGGER.debug('first_file_id :%s date: %s', first_file_id, date)
        return first_file_id, cursor 

def get_input(self):
        """ Loads web input, initialise default values and check/sanitise some inputs from users """
        user_input = web.input(
            users=[],
            tasks=[],
            audiences=[],
            org_tags=[],
            grade_min='',
            grade_max='',
            sort_by="submitted_on",
            order='0',  # "0" for pymongo.DESCENDING, anything else for pymongo.ASCENDING
            limit='',
            filter_tags=[],
            filter_tags_presence=[],
            date_after='',
            date_before='',
            stat='with_stat',
        )

        # Sanitise inputs
        for item in itertools.chain(user_input.tasks, user_input.audiences):
            if not id_checker(item):
                raise web.notfound()

        if user_input.sort_by not in self._allowed_sort:
            raise web.notfound()

        digits = [user_input.grade_min, user_input.grade_max, user_input.order, user_input.limit]
        for d in digits:
            if d != '' and not d.isdigit():
                raise web.notfound()

        return user_input 

def get_coll(client=None):
    cache = QA_util_cache()
    results = cache.get('tdx_coll')
    if results:
        return results
    else:
        _coll = client.index_day
        _coll.create_index(
            [('code',
              pymongo.ASCENDING),
             ('date_stamp',
              pymongo.ASCENDING)]
        )
        cache.set('tdx_coll', _coll, age=86400)
        return _coll 

def get_owned_assets(addresses):
    """Gets a list of owned assets for one or more addresses"""
    result = config.mongo_db.tracked_assets.find({
        'owner': {"$in": addresses}
    }, {"_id": 0}).sort("asset", pymongo.ASCENDING)
    return list(result) 

def ensure_indexes(self):
        #Ensure index existence
        db = self.client.bitcoin
        collection = db.transactions
        collection.create_index([("source_n_id", ASCENDING)])
        collection.create_index([("destination_n_id", ASCENDING)])
        collection.create_index([("source", ASCENDING)])
        collection.create_index([("destination", ASCENDING)])
        collection.create_index([("block_id",DESCENDING)]) 

def get_oplog_head_ts(self):
        logging.debug("Gathering oldest 'ts' in %s oplog" % self.uri)
        return self.get_oplog_rs().find_one(sort=[('$natural', ASCENDING)])['ts'] 

def init():
    # init db and indexes
    # asset_extended_info
    config.mongo_db.asset_extended_info.ensure_index('asset', unique=True)
    config.mongo_db.asset_extended_info.ensure_index('info_status')
    # balance_changes
    config.mongo_db.balance_changes.ensure_index('block_index')
    config.mongo_db.balance_changes.ensure_index([
        ("address", pymongo.ASCENDING),
        ("asset", pymongo.ASCENDING),
        ("block_index", pymongo.DESCENDING),
        ("_id", pymongo.DESCENDING)
    ])
    #config.mongo_db.balance_changes.ensure_index([
    #    ("address", pymongo.ASCENDING),
    #    ("asset_longname", pymongo.ASCENDING),
    #])
    try:  # drop unnecessary indexes if they exist
        config.mongo_db.balance_changes.drop_index('address_1_asset_1_block_time_1')
    except:
        pass

    # tracked_assets
    config.mongo_db.tracked_assets.ensure_index('asset', unique=True)
    config.mongo_db.tracked_assets.ensure_index('_at_block')  # for tracked asset pruning
    config.mongo_db.tracked_assets.ensure_index([
        ("owner", pymongo.ASCENDING),
        ("asset", pymongo.ASCENDING),
    ])
    # feeds (also init in betting module)
    config.mongo_db.feeds.ensure_index('source')
    config.mongo_db.feeds.ensure_index('owner')
    config.mongo_db.feeds.ensure_index('category')
    config.mongo_db.feeds.ensure_index('info_url') 

def init():
    # init db and indexes
    # transaction_stats
    config.mongo_db.transaction_stats.ensure_index([  # blockfeed.py, api.py
        ("when", pymongo.ASCENDING),
        ("category", pymongo.DESCENDING)
    ])
    config.mongo_db.transaction_stats.ensure_index('block_index') 

def updateDays(self, code, data):
        """ @data: [{row0}, {row1}] """
        # create index
        for _ in range(3):
            try:
                collection = self._getStockDaysDb()[code]
                collection.create_index([('datetime', pymongo.ASCENDING)], unique=True)
                break
            except pymongo.errors.AutoReconnect as ex:
                lastEx = ex
                print("更新{}日线数据到MongoDB异常: {}".format(code, str(ex) + ', ' + str(ex.details)))
                sleep(1)
        else:
            self._info.print("更新{}日线数据到MongoDB异常: {}".format(code, str(lastEx) + ', ' + str(lastEx.details)), DyLogData.error)
            return False

        # update to DB
        try:
            for doc in data:
                flt = {'datetime': doc['datetime']}
                collection.update_one(flt, {'$set':doc}, upsert=True)
        except Exception as ex:
            self._info.print("更新{0}日线数据到MongoDB异常:{1}".format(code, str(ex) + ', ' + str(ex.details)), DyLogData.error)
            return False

        return True 

def _findOneCodeDaysByRelative(self, code, baseDate, n=0, name=None):
        """
            包含当日，也就是说offset 0总是被包含的
        """
        # 获取当日日期
        baseDay = self._getCodeDay(code, baseDate, name)
        if baseDay is None: return None

        collection = self._getStockDaysDb()[code]

        if n <= 0:
            date = datetime.strptime(baseDay + ' 23:00:00', '%Y-%m-%d %H:%M:%S')
            flt = {'datetime':{'$lt':date}}

            sortMode = pymongo.DESCENDING
        else:
            date = datetime.strptime(baseDay, '%Y-%m-%d')
            flt = {'datetime':{'$gte':date}} # ignore baseDate, no matter its in DB or not

            sortMode = pymongo.ASCENDING

        # 向前贪婪
        n = abs(n) + 1

        try:
            cursor = collection.find(flt).sort('datetime', sortMode).limit(n)
        except Exception as ex:
            self._info.print("MongoDB Exception({0}): @_findOneCodeDaysByRelative{1}:{2}, [{3}, {4}]日线数据".format(str(ex) + ', ' + str(ex.details),
                                                                                                                    code, name,
                                                                                                                    baseDate, n),
                             DyLogData.error)
            return None

        # We don't check any thing about if we actually get n days data.
        # The reason is that we don't know future, as well as 何时股票上市
        
        return cursor 

def _getTradeDaysByRelativePositive(self, baseDate, n):

        baseDateSave = baseDate
        nSave = n

        # always get 0 offset trade day
        baseDate = self._getTradeDaysByRelativeZero(baseDate)
        if baseDate is None: return None

        # find backward n trade days
        collection = self._getTradeDayTableCollection()

        flt = {'datetime': {'$gt': baseDate[0]['datetime']}}

        try:
            cursor = collection.find(flt).sort('datetime', pymongo.ASCENDING)
        except Exception as ex:
            self._info.print("MongoDB Exception({0}): @_getTradeDaysByRelativePositive({1}, {2})".format(str(ex) + ', ' + str(ex.details), baseDateSave, nSave),
                             DyLogData.error)
            return None

        dates = [baseDate[0]]
        for d in cursor:
            if d['tradeDay']:
                dates.append(d)

                n -= 1
                if n == 0:
                    return dates

        # 如果数据库里的最新日期不是今日，提醒更新数据, 并返回None
        date = self.getDaysLatestDate()
        if date is not None:
            now = datetime.now()
            if now > datetime(now.year, now.month, now.day, 18, 0, 0) and DyTime.dateCmp(date['datetime'], now) != 0:
                self._info.print("数据库里的最新日期不是今日, 请更新历史日线数据", DyLogData.error)

                return None

        return dates