Python google.appengine.ext.ndb.delete_multi() Examples

def testGetVotes_Inactive(self):
    self.assertLen(self.blockable_1.GetVotes(), 0)

    test_utils.CreateVotes(self.blockable_1, 2)

    self.assertLen(self.blockable_1.GetVotes(), 2)

    votes = vote_models.Vote.query().fetch()
    new_votes = []
    for vote in votes:
      new_key = ndb.Key(flat=vote.key.flat()[:-1] + (None,))
      new_votes.append(datastore_utils.CopyEntity(vote, new_key=new_key))
    ndb.delete_multi(vote.key for vote in votes)
    ndb.put_multi(new_votes)

    self.assertLen(self.blockable_1.GetVotes(), 0) 

def task_delete_tasks(task_ids):
  """Deletes the specified tasks, a list of string encoded task ids."""
  total = 0
  count = 0
  opt = ndb.QueryOptions(use_cache=False, use_memcache=False, keys_only=True)
  try:
    for task_id in task_ids:
      request_key = task_pack.unpack_request_key(task_id)
      # Delete the whole group. An ancestor query will retrieve the entity
      # itself too, so no need to explicitly delete it.
      keys = ndb.Query(default_options=opt, ancestor=request_key).fetch()
      if not keys:
        # Can happen if it is a retry.
        continue
      ndb.delete_multi(keys)
      total += len(keys)
      count += 1
    return count
  finally:
    logging.info(
        'Deleted %d TaskRequest groups; %d entities in total', count, total) 

def PutResource(url, etag, content):
  """Persist a resource."""
  key = ndb.Key(Resource, url, namespace=settings.PLAYGROUND_NAMESPACE)
  keys = ndb.Query(ancestor=key).fetch(keys_only=True)
  ndb.delete_multi(keys)
  resource = Resource(id=url, etag=etag,
                      namespace=settings.PLAYGROUND_NAMESPACE)
  if len(content) <= _MAX_RAW_PROPERTY_BYTES:
    resource.content = content
    resource.put()
    return
  chunks = [content[i:i + _MAX_RAW_PROPERTY_BYTES]
            for i in range(0, len(content), _MAX_RAW_PROPERTY_BYTES)]
  entities = [ResourceChunk(id=i + 1, parent=resource.key, content=chunks[i])
              for i in range(0, len(chunks))]
  entities.append(resource)
  ndb.put_multi(entities) 

def DeleteReposAndTemplateProjects():
  """Delete repos and related template projects."""
  user = GetPublicTemplateOwner()

  # delete template projects
  keys = user.projects
  ndb.delete_multi(keys)

  # delete ANONYMOUS user
  user.key.delete()

  # delete code repositories
  query = Repo.query(namespace=settings.PLAYGROUND_NAMESPACE)
  keys = query.fetch(keys_only=True)
  ndb.delete_multi(keys)

  # delete repo collections
  query = RepoCollection.query(ancestor=GetGlobalRootEntity().key)
  keys = query.fetch(keys_only=True)
  ndb.delete_multi(keys)

  # flush memcache
  memcache.flush_all() 

def account_cleanup(stars, cursor=None):
  account_qry = model.Account.query().filter(model.Account.stars < stars)
  account_keys, account_cursors = util.get_dbs(
      account_qry,
      order='stars',
      keys_only=True,
      cursor=cursor,
    )

  ndb.delete_multi(account_keys)
  if account_cursors['next']:
    deferred.defer(account_cleanup, days, account_cursors['next'])


###############################################################################
# Rank Accounts
############################################################################### 

def repo_cleanup(days, cursor=None):
  before_date = datetime.utcnow() - timedelta(days=days)
  repo_qry = model.Repo.query().filter(model.Repo.modified < before_date)
  repo_keys, repo_cursors = util.get_dbs(
      repo_qry,
      order='modified',
      keys_only=True,
      cursor=cursor,
    )

  ndb.delete_multi(repo_keys)
  if repo_cursors['next']:
    deferred.defer(repo_cleanup, days, repo_cursors['next'])


###############################################################################
# Account Clean-ups
############################################################################### 

def DeleteProject(project):
  """Delete an existing project."""
  assert project
  tree = _CreateProjectTree(project)
  # 1. delete files
  tree.Clear()

  @ndb.transactional(xg=True)
  def DelProject():
    # 2. get current entities
    prj = project.key.get()
    user_key = ndb.Key(User, prj.owner, namespace=settings.PLAYGROUND_NAMESPACE)
    usr = user_key.get()
    # 3. delete project
    prj.key.delete()
    # 4. delete project references
    if prj.key in usr.projects:
      usr.projects.remove(prj.key)
      usr.put()
    else:
      shared.i('ignoring project key {} not found in user projects {}'
               .format(prj.key, usr.projects))

  @ndb.transactional(xg=True)
  def DelReposAndProject(keys):
    ndb.delete_multi(keys)
    DelProject()

  repo_query = Repo.query(namespace=settings.PLAYGROUND_NAMESPACE)
  repo_query = repo_query.filter(Repo.project == project.key)
  keys = repo_query.fetch(keys_only=True)
  if keys:
    DelReposAndProject(keys)
  else:
    DelProject() 

def _flush_ndb_deletes(self, items, options):
    """Flush all deletes to datastore."""
    assert ndb is not None
    ndb.delete_multi(items, config=self._create_config(options)) 

def _flush_ndb_deletes(self, items, options):
    """Flush all deletes to datastore."""
    assert ndb is not None
    ndb.delete_multi(items, config=self._create_config(options)) 

def tearDown(self):
    ndb.delete_multi(schema.LastPoints.query().fetch(keys_only=True))
    self.testbed.deactivate() 

def get(self):
        logging.debug('ExpireMemberCandidates.get hit')
        now = datetime.datetime.now()
        expireds = MemberCandidate.query(MemberCandidate.expire <= now).fetch()
        if expireds:
            logging.info('Expiring %d MemberCandidate items', len(expireds))
            ndb.delete_multi([expired.key for expired in expireds]) 

def love_links_cleanup():
    """
    Deletes love links that are more than a month (30 days) old.
    """
    earliest = datetime.datetime.now() - datetime.timedelta(days=30)
    love_links_keys = LoveLink.query(LoveLink.timestamp <= earliest).fetch(keys_only=True)
    logging.info('Preparing to delete love links older than {}.'.format(str(earliest)))
    ndb.delete_multi(love_links_keys)
    logging.info('Love links older than {} were deleted.'.format(str(earliest)))

    return 

def rebuild_love_count():
    utc_dt = datetime.datetime.utcnow() - datetime.timedelta(days=7)  # rebuild last week and this week
    week_start, _ = utc_week_limits(utc_dt)

    set_toggle_state(LOVE_SENDING_ENABLED, False)

    logging.info('Deleting LoveCount table... {}MB'.format(memory_usage().current()))
    ndb.delete_multi(LoveCount.query(LoveCount.week_start >= week_start).fetch(keys_only=True))
    employee_dict = {
        employee.key: employee
        for employee in Employee.query()
    }
    logging.info('Rebuilding LoveCount table... {}MB'.format(memory_usage().current()))
    cursor = None
    count = 0
    while True:
        loves, cursor, has_more = Love.query(Love.timestamp >= week_start).fetch_page(500, start_cursor=cursor)
        for l in loves:
            LoveCount.update(l, employee_dict=employee_dict)
        count += len(loves)
        logging.info('Processed {} loves, {}MB'.format(count, memory_usage().current()))
        if not has_more:
            break
    logging.info('Done. {}MB'.format(memory_usage().current()))

    set_toggle_state(LOVE_SENDING_ENABLED, True) 

def remove_unique(cls, auth_id, unique_properties, **user_values):
        uniques = [('%s.auth_id:%s' % (cls.__name__, auth_id), 'auth_id')]
        print uniques
        if unique_properties:
            for name in unique_properties:
                key = '%s.%s:%s' % (cls.__name__, name, user_values[name])
                uniques.append((key, name))

        # Delete the uniques
        ndb.delete_multi(model.Key(cls.unique_model, k) for k,v in uniques) 

def main():
    user = users.get_current_user()
    if not user:
        return flask.redirect(users.create_login_url(flask.request.path))

    if flask.request.method == 'POST':
        util.csrf_protect()
        tab_ids = flask.request.values.getlist('tab_id')
        keys = [ndb.Key(Session, tab_id) for tab_id in tab_ids]
        if 'delete' in flask.request.values:
            if not all(s and s.user_id == user.user_id() for s in ndb.get_multi(keys)):
                return 'Not authorized to delete some sessions', 403
            ndb.delete_multi(keys)
        elif 'share' in flask.request.values:
            for key in keys:
                session = key.get()
                if session and session.user_id == user.user_id():
                    session.shared = True
                    session.put()
        else:
            return 'Incorrect POST name', 400
    date = flask.request.values.get('date', datetime.now().strftime('%Y-%m-%d'))
    cur_day = datetime.strptime(date, '%Y-%m-%d')
    next_day = cur_day + timedelta(days=1)
    sessions = (Session.query(Session.user_id == user.user_id(),
                Session.start_ts >= cur_day, Session.start_ts < next_day)
            .order(-Session.start_ts))
    num_shared = Session.query(Session.user_id == user.user_id(), Session.shared == True).count()
    return flask.render_template('main.html',
                                 user=user,
                                 date=date,
                                 year=datetime.now().year,
                                 logout_url=users.create_logout_url('/'),
                                 sessions=sessions,
                                 num_shared=num_shared) 

def get(self):
    old_cutoff = utils.utcnow() - on_error.ERROR_TIME_TO_LIVE
    items = models.Error.query(
        models.Error.created_ts < old_cutoff,
        default_options=ndb.QueryOptions(keys_only=True))
    out = len(ndb.delete_multi(items))
    self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
    self.response.write(str(out))


### Public API. 

def revoke_stale_authorization():
  """Removes authorization from accounts that no longer have access."""
  to_delete = []
  for email in _list_authorized_readers():
    ident = auth.Identity.from_bytes('user:' + email)
    if not acl.is_trusted_service(ident):
      logging.warning('Removing "%s" as authorized GCS reader', email)
      to_delete.append(_auth_db_reader_key(email))
  ndb.delete_multi(to_delete)
  # Update ACLs even if we didn't delete anything. This is necessary to make
  # revoke_stale_authorization() idempotent: even if it crashes right after
  # ndb.delete_multi, we still will remove stale GCS ACLs on a retry.
  _update_gcs_acls() 

def get(self):
    old_cutoff = utils.utcnow() - on_error.ERROR_TIME_TO_LIVE
    items = models.Error.query(
        models.Error.created_ts < old_cutoff,
        default_options=ndb.QueryOptions(keys_only=True))
    out = len(ndb.delete_multi(items))
    self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
    self.response.write(str(out))


### Public API. 

def get(self):
    old_cutoff = utils.utcnow() - on_error.ERROR_TIME_TO_LIVE
    items = models.Error.query(
        models.Error.created_ts < old_cutoff,
        default_options=ndb.QueryOptions(keys_only=True))
    out = len(ndb.delete_multi(items))
    self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
    self.response.write(str(out))


### Public API. 

def test_bot_event_poll_sleep(self):
    _bot_event(event_type='request_sleep')

    # Assert that BotInfo was updated too.
    expected = _gen_bot_info(composite=[
        bot_management.BotInfo.NOT_IN_MAINTENANCE,
        bot_management.BotInfo.ALIVE,
        bot_management.BotInfo.HEALTHY,
        bot_management.BotInfo.IDLE,
    ])

    bot_info = bot_management.get_info_key('id1').get()
    self.assertEqual(expected, bot_info.to_dict())

    # BotEvent is registered for poll when BotInfo creates
    expected_event = _gen_bot_event(event_type=u'request_sleep')
    bot_events = bot_management.get_events_query('id1', True)
    self.assertEqual([expected_event], [e.to_dict() for e in bot_events])

    # flush bot events
    ndb.delete_multi(e.key for e in bot_events)

    # BotEvent is not registered for poll when no dimensions change
    _bot_event(event_type='request_sleep')
    self.assertEqual([], bot_management.get_events_query('id1', True).fetch())

    # BotEvent is registered for poll when dimensions change
    dims = {u'foo': [u'bar']}
    _bot_event(event_type='request_sleep', dimensions=dims)
    expected_event['dimensions'] = dims
    bot_events = bot_management.get_events_query('id1', True).fetch()
    self.assertEqual([expected_event], [e.to_dict() for e in bot_events]) 

def test_list_tags_none(self):
    ndb.delete_multi(tag_model.Tag.query().fetch(keys_only=True))
    (query_results, cursor,
     has_additional_results), total_pages = tag_model.Tag.list()
    self.assertEmpty(query_results)
    self.assertIsNone(cursor)
    self.assertFalse(has_additional_results)
    self.assertEqual(total_pages, 0) 

def cron_delete_old_bot():
  """Deletes stale BotRoot entity groups."""
  start = utils.utcnow()
  # Run for 4.5 minutes and schedule the cron job every 5 minutes. Running for
  # 9.5 minutes (out of 10 allowed for a cron job) results in 'Exceeded soft
  # private memory limit of 512 MB with 512 MB' even if this loop should be
  # fairly light on memory usage.
  time_to_stop = start + datetime.timedelta(seconds=int(4.5*60))
  total = 0
  deleted = []
  try:
    q = BotRoot.query(default_options=ndb.QueryOptions(keys_only=True))
    for bot_root_key in q:
      # Check if it has any BotEvent left. If not, it means that the entity is
      # older than _OLD_BOT_EVENTS_CUF_OFF, so the whole thing can be deleted
      # now.
      # In particular, ignore the fact that BotInfo may still exist, since if
      # there's no BotEvent left, it's probably a broken entity or a forgotten
      # dead bot.
      if BotEvent.query(ancestor=bot_root_key).count(limit=1):
        continue
      deleted.append(bot_root_key.string_id())
      # Delete the whole group. An ancestor query will retrieve the entity
      # itself too, so no need to explicitly delete it.
      keys = ndb.Query(ancestor=bot_root_key).fetch(keys_only=True)
      ndb.delete_multi(keys)
      total += len(keys)
      if utils.utcnow() >= time_to_stop:
        break
    return total
  except runtime.DeadlineExceededError:
    pass
  finally:
    logging.info(
        'Deleted %d entities from the following bots:\n%s',
        total, ', '.join(sorted(deleted))) 

def get(self):
    old_cutoff = utils.utcnow() - on_error.ERROR_TIME_TO_LIVE
    items = models.Error.query(
        models.Error.created_ts < old_cutoff,
        default_options=ndb.QueryOptions(keys_only=True))
    out = len(ndb.delete_multi(items))
    self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
    self.response.write(str(out))


### Public API. 

def test_generate_meeting_specs(database):
    # delete current specs
    keys = [key for key in MeetingSpec.query().iter(keys_only=True)]
    ndb.delete_multi(keys)
    meeting_specs = MeetingSpec.query().fetch()
    assert len(meeting_specs) == 0

    # ensure we create new specs
    generate_meeting_specs()
    meeting_specs = MeetingSpec.query().fetch()
    assert len(meeting_specs) == 2 

def create_batch(cls, size, **kwargs):
        """Firstly deletes all entries from datastore and then
         creates batch of new entries"""
        ndb.delete_multi(cls._meta.model.query().fetch(keys_only=True))
        super(BaseFactory, cls).create_batch(size, **kwargs) 

def test_phone_numbers(contact_key):
    """A test for 'phone_numbers' property."""
    models.PhoneNumber(parent=contact_key,
                       phone_type='home',
                       number='(650) 555 - 2200').put()
    models.PhoneNumber(parent=contact_key,
                       phone_type='mobile',
                       number='(650) 555 - 2201').put()
    contact = contact_key.get()
    for phone in contact.phone_numbers:
        # it doesn't ensure any order
        if phone.phone_type == 'home':
            assert '(650) 555 - 2200' == phone.number
        elif phone.phone_type == 'mobile':
            assert phone.number == '(650) 555 - 2201'

    # filer the phone numbers by type. Note that this is an
    # ancestor query.
    query = contact.phone_numbers.filter(
        models.PhoneNumber.phone_type == 'home')
    entities = query.fetch()
    assert 1 == len(entities)
    assert entities[0].number == '(650) 555 - 2200'

    # delete the mobile phones
    query = contact.phone_numbers.filter(
        models.PhoneNumber.phone_type == 'mobile')
    ndb.delete_multi([e.key for e in query])

    # make sure there's no mobile phones any more
    query = contact.phone_numbers.filter(
        models.PhoneNumber.phone_type == 'mobile')
    entities = query.fetch()
    assert 0 == len(entities) 

def operate_on_multiple_keys_at_once(list_of_entities):
    list_of_keys = ndb.put_multi(list_of_entities)
    list_of_entities = ndb.get_multi(list_of_keys)
    ndb.delete_multi(list_of_keys) 

def delete_entries():
    return ndb.delete_multi(CronLog.query().fetch(keys_only=True))


# Parameters
# site: site url string
# Returns
# gsc-style date or never string 

def delete_old_news():
    q = NewsEntry.query(NewsEntry.important == True).order(-NewsEntry.published)
    ndb.delete_multi(q.fetch(offset=app.config["PER_PAGE"], keys_only=True))
    q = NewsEntry.query(NewsEntry.important == False).order(-NewsEntry.published)
    ndb.delete_multi(q.fetch(offset=app.config["PER_PAGE"], keys_only=True))
    return "Done", 200 

def FlushAllCaches():
    """Removes any cached data from datastore/memache."""
    chart_data_keys = ChartData.query().fetch(keys_only=True)
    ndb.delete_multi(chart_data_keys)
    project_list_keys = Projects.query().fetch(keys_only=True)
    ndb.delete_multi(project_list_keys)