Python pymongo.Connection() Examples

def after_login(self, response):
        connection = pymongo.Connection("localhost", 27017)
        self.db = connection["zhihu"]
        self.zh_user_col = self.db["zh_user"]

        for key in ["高新科技", "互联网", "电子商务", "电子游戏", "计算机软件", "计算机硬件"]:
            users = self.zh_user_col.find({"industry": key})
            # print users.count()
            for user in users:
                # answers
                num = int(user['answer_num']) if "answer_num" in user.keys()  else 0
                page_num = num / 20
                page_num += 1 if num % 20 else 0
                for i in xrange(page_num):
                    yield Request(host + "/people/" + user["username"] + '/answers?page=%d' % (i + 1),
                                  callback=self.parse_ans) 

def after_login(self, response):
        connection = pymongo.Connection("localhost", 27017)
        self.db = connection["zhihu"]
        self.zh_user_col = self.db["zh_user"]

        for key in ["高新科技", "互联网", "电子商务", "电子游戏", "计算机软件", "计算机硬件"]:
            users = self.zh_user_col.find({"industry": key})
            # print users.count()
            for user in users:
                # questions
                num = int(user['ask_num']) if "ask_num" in user.keys() else 0
                page_num = num / 20
                page_num += 1 if num % 20 else 0
                for i in xrange(page_num):
                    url = host + "/people/" + user["username"] + '/asks?page=%d' % (i + 1)
                    yield Request(url, callback=self.parse_ask) 

def depiler(pile, pile_deleted, pile_catchup, pile_medias, mongoconf, locale, exit_event, debug=False):
    db = MongoClient(mongoconf['host'], mongoconf['port'])[mongoconf['db']]
    coll = db['tweets']
    while not exit_event.is_set() or not pile.empty() or not pile_deleted.empty():
        while not pile_deleted.empty():
            todelete = pile_deleted.get()
            coll.update({'_id': todelete}, {'$set': {'deleted': True}}, upsert=True)

        todo = []
        while not pile.empty():
            todo.append(pile.get())
        stored = 0
        for t in prepare_tweets(todo, locale):
            if pile_medias and t["medias"]:
                pile_medias.put(t)
            if pile_catchup and t["in_reply_to_status_id_str"]:
                if not coll.find_one({"_id": t["in_reply_to_status_id_str"]}):
                    pile_catchup.put(t["in_reply_to_status_id_str"])
            tid = coll.update({'_id': t['_id']}, {'$set': t}, upsert=True)
            stored += 1
        if debug and stored:
            log("DEBUG", "Saved %s tweets in MongoDB" % stored)
        breakable_sleep(2, exit_event)
    log("INFO", "FINISHED depiler") 

def __connectMongo(self):
        """连接MongoDB数据库"""
        try:
            self.__mongoConnection = Connection()
            self.__mongoConnected = True
            self.__mongoTickDB = self.__mongoConnection['TickDB']
            self.writeLog(u'策略引擎连接MongoDB成功')
        except ConnectionFailure:
            self.writeLog(u'策略引擎连接MongoDB失败')

    #---------------------------------------------------------------------- 

def __init__(self):
        params = {k: getattr(settings, k, v) for k, v in param_template}
        # running mode detect by media_path
        self.repo_path = params['REPO_PATH'] or ''
        if self.repo_path is not None and (not os.path.exists(self.repo_path)):
            os.makedirs(self.repo_path)
        self.hasRepo = os.path.exists(self.repo_path)

        # connect to db
        self.conn = pymongo.Connection(params['host'], int(params['port']))
        self.db = self.conn[params['db_name']] 

def __init__(self, *a, **kwargs):
        super(GithubUserSpider, self).__init__(*a, **kwargs)
        import pymongo
        connection = pymongo.Connection("localhost", 27017)
        self.db = connection["zhihu"]
        self.gh_user_col = self.db["gh_user"] 

def __init__(self, col, index=None):
        connection = pymongo.Connection(settings.MONGODB_SERVER, settings.MONGODB_PORT)
        self.db = connection[settings.MONGODB_DB]
        self.collection = self.db[col]
        if index:
            self.collection.create_index(index, unique=True) 

def __init__(self):
        import pymongo
        connection = pymongo.Connection(settings['MONGODB_SERVER'], settings['MONGODB_PORT'])
        self.db = connection[settings['MONGODB_DB']]
        self.collection = self.db[settings['MONGODB_COLLECTION']]
        if self.__get_uniq_key() is not None:
            self.collection.create_index(self.__get_uniq_key(), unique=True) 

def __init__(self):
        handlers = [
            (r"/", MainHandler),
            (r"/recommended/", RecommendedHandler),
        ]
        settings = dict(
            template_path=os.path.join(os.path.dirname(__file__), "templates"),
            static_path=os.path.join(os.path.dirname(__file__), "static"),
            ui_modules={"Book": BookModule},
            debug=True,
        )
        conn = pymongo.Connection("localhost", 27017)
        self.db = conn["bookstore"]
        tornado.web.Application.__init__(self, handlers, **settings) 

def mongodb_stats(host, p, database, username, password):
    port = int(p)
    try:
        if username and password and database:
            c = Client("mongodb://"+username+":"+password+"@"+host+"/"+database, port)
        elif username and password:
            c = Client('mongodb://'+username+':'+password+'@'+host+'/', port)
        elif database:
            c = Client('mongodb://'+host+'/'+database, port)
        else:
            c = Client(host, port)
    except ConnectionFailure, AutoReconnect:
        return None 

def nosql():
    import pymongo
    import json
    conn = pymongo.Connection('127.0.0.1', port=27017)
    df = ts.get_tick_data('600848',date='2014-12-22')
    print(df.to_json(orient='records'))
    
    conn.db.tickdata.insert(json.loads(df.to_json(orient='records')))
    
#     print conn.db.tickdata.find() 

def __init__(self, host="localhost", port=27017, database="snakemq",
                  collection="snakemq"):
        self.conn = pymongo.Connection(host, port)
        self.database = self.conn[database]
        self.collection = self.database[collection]
        self.all_items = self.collection.snakemq_items

    #################################################### 

def __connectMongo(self):
        """连接MongoDB数据库"""
        try:
            self.__mongoConnection = Connection()
            self.__mongoConnected = True
            self.__mongoTickDB = self.__mongoConnection['TickDB']
            self.writeLog(u'策略引擎连接MongoDB成功')
        except ConnectionFailure:
            self.writeLog(u'策略引擎连接MongoDB失败')

    #---------------------------------------------------------------------- 

def connectMongo(self):
        """连接MongoDB数据库"""
        try:
            self.__mongoConnection = Connection()
            self.__mongoConnected = True
            self.__mongoTickDB = self.__mongoConnection['TickDB']
            self.writeLog(u'回测引擎连接MongoDB成功')
        except ConnectionFailure:
            self.writeLog(u'回测引擎连接MongoDB失败') 
            
    #---------------------------------------------------------------------- 

def get_connection():
    global _connection
    if not _connection:
        _connection = Connection(
            host=getattr(settings, 'MONGODB_HOST', None),
            port=getattr(settings, 'MONGODB_PORT', None)
        )
        username = getattr(settings, 'MONGODB_USERNAME', None)
        password = getattr(settings, 'MONGODB_PASSWORD', None)
        db = _connection[settings.MONGODB_DATABASE]
        if username and password:
            db.authenticate(username, password)
        return db
    return _connection[settings.MONGODB_DATABASE] 

def connectMongo(self):
        """连接MongoDB数据库"""
        try:
            self.__mongoConnection = Connection()
            self.__mongoConnected = True
            self.__mongoTickDB = self.__mongoConnection['TickDB']
            self.writeLog(u'回测引擎连接MongoDB成功')
        except ConnectionFailure:
            self.writeLog(u'回测引擎连接MongoDB失败') 
            
    #---------------------------------------------------------------------- 

def connect_to_db_collection(db_name, collection_name):
    '''
    Return collection of a given database name and collection name
    
    '''
    connection = pymongo.Connection('localhost', 27017)
    db = connection[db_name]
    collection = db[collection_name]
    return collection 

def connect_to_db_collection(db, collection):
    db_name = db
    collection_name = collection
    
    # Get database connection and collections
    connection = pymongo.Connection('localhost', 27017)
    db = connection[db_name]
    tracking = db[collection_name]
    tracking_imported = db[collection_name + "_imported"]
    return tracking, tracking_imported 

def connect_to_db_collection(db_name, collection_name):
    '''
    Retrieve collection from given database name and collection name

    '''
    connection = pymongo.Connection('localhost', 27017)
    db = connection[db_name]
    collection = db[collection_name]
    return collection 

def connect_to_db_collection(db_name, collection_name):
    '''
    Return collection of a given database name and collection name
    
    '''
    connection = pymongo.Connection('localhost', 27017)
    db = connection[db_name]
    collection = db[collection_name]
    return collection 

def connect_to_db_collection(db_name, collection_name):
    '''
    Retrieve collection from given database name and collection name

    '''
    connection = pymongo.Connection('localhost', 27017)
    db = connection[db_name]
    collection = db[collection_name]
    return collection 

def nosql():
    import pymongo
    import json
    conn = pymongo.Connection('127.0.0.1', port=27017)
    df = ts.get_tick_data('600848',date='2014-12-22')
    print(df.to_json(orient='records'))
    
    conn.db.tickdata.insert(json.loads(df.to_json(orient='records')))
    
#     print conn.db.tickdata.find() 

def __init__(self):
        conn = pymongo.Connection(
            settings['MONGO_CONF']['host'],
            settings['MONGO_CONF']['port']
        )
        db = conn[settings['MONGO_CONF']['db']]
        self.subscription_collection = db[settings['MONGO_CONF']['subscription_collection']] 

def __init__(self):
        conn = pymongo.Connection(
            settings['MONGO_CONF']['host'],
            settings['MONGO_CONF']['port']
        )
        db = conn[settings['MONGO_CONF']['db']]
        self.news_collection = db[settings['MONGO_CONF']['collection']] 

def __init__(self):
        self.server = settings['MONGODB_SERVER']
        self.port = settings['MONGODB_PORT']
        self.db = settings['MONGODB_DB']
        self.col = settings['MONGODB_COLLECTION']
        connection = pymongo.Connection(self.server, self.port)
        db = connection[self.db]
        self.collection = db[self.col] 

def setUp(self):
        self.client = pymongo.Connection(os.environ.get("TEST_MONGODB"))
        self.db = self.client.test_queue
        self.queue = MongoQueue(self.db.queue_1, "consumer_1") 

def setUp(self):
        self.client = pymongo.Connection(os.environ.get("TEST_MONGODB"))
        self.db = self.client.test_queue
        self.collection = self.db.locks 

def __init__(self, host = db_config['DB_HOST'], 
                        port = db_config['DB_PORT'],
                        user = db_config['DB_USER'], 
                        passwd = db_config['DB_PSW'], 
                        db = db_config['DB_NAME'], 
                        charset = db_config['DB_CHARSET']):
        
        self.connection = pymongo.Connection(host, port)
        self.db = self.connection[db]
        self.db.authenticate(user, passwd) 

def __init__(self, **kwargs):
        super(MongoDB, self).__init__(**kwargs)
        if self._connection is None:
            self._connection = Connection(*self.connection_args, **self.connection_kwargs)
        if not self.database:
            self.database = self.session
        self._db = self._connection[self.database]
        self._records = self._db['task_records']
        self._records.ensure_index('msg_id', unique=True)
        self._records.ensure_index('submitted') # for sorting history
        # for rec in self._records.find 

def resolver(mongoconf, exit_event, debug=False):
    ua = UserAgent()
    ua.update()
    db = MongoClient(mongoconf['host'], mongoconf['port'])[mongoconf['db']]
    linkscoll = db['links']
    tweetscoll = db['tweets']
    while not exit_event.is_set():
        done = 0
        todo = list(tweetscoll.find({"links_to_resolve": True}, projection={"links": 1, "proper_links": 1, "retweet_id": 1}, limit=600, sort=[("_id", 1)]))
        urlstoclear = list(set([l for t in todo if not t.get("proper_links", []) for l in t.get('links', [])]))
        alreadydone = {l["_id"]: l["real"] for l in linkscoll.find({"_id": {"$in": urlstoclear}})}
        tweetsdone = []
        batchidsdone = set()
        for tweet in todo:
            if tweet.get("proper_links", []):
                tweetsdone.append(tweet["_id"])
                continue
            tweetid = tweet.get('retweet_id') or tweet['_id']
            if tweetid in batchidsdone:
                continue
            if exit_event.is_set():
                continue
            gdlinks = []
            for link in tweet.get("links", []):
                if link in alreadydone:
                    gdlinks.append(alreadydone[link])
                    continue
                good = resolve_url(link, user_agent=ua)
                gdlinks.append(good)
                alreadydone[link] = good
                try:
                    linkscoll.save({'_id': link, 'real': good})
                except Exception as e:
                    log("WARNING", "Could not store resolved link %s -> %s because %s: %s" % (link, good, type(e), e))
                if link != good:
                    done += 1
            tweetscoll.update({'$or': [{'_id': tweetid}, {'retweet_id': tweetid}]}, {'$set': {'proper_links': gdlinks, 'links_to_resolve': False}}, upsert=False, multi=True)
            batchidsdone.add(tweetid)
        if debug and done:
            left = tweetscoll.count({"links_to_resolve": True})
            log("DEBUG", "[links] +%s new redirection resolved out of %s links (%s waiting)" % (done, len(todo), left))
        # clear tweets potentially rediscovered
        if tweetsdone:
            tweetscoll.update({"_id": {"$in": tweetsdone}}, {"$set": {"links_to_resolve": False}}, upsert=False, multi=True)
    log("INFO", "FINISHED resolver")