Python google.appengine.api.memcache.set() Examples

def get_by_sid(cls, sid):
        """Returns a ``Session`` instance by session id.

        :param sid:
            A session id.
        :returns:
            An existing ``Session`` entity.
        """
        data = memcache.get(sid)
        if not data:
            session = model.Key(cls, sid).get()
            if session:
                data = session.data
                memcache.set(sid, data)

        return data 

def get(self):
    util.EnableCors(self)

    WP_PLEDGES = 4099
    VERSION_12_AND_UNDER = 59009 

    count = memcache.get('TOTAL-PLEDGES')
    if not count:
      query = model.Pledge.all(keys_only=True).filter('model_version >', 12)
      i = 0
      while True:
          result = query.fetch(1000)
          i = i + len(result)
          if len(result) < 1000:
              break
          cursor = query.cursor()
          query.with_cursor(cursor)
      count = i + WP_PLEDGES + VERSION_12_AND_UNDER
      memcache.set('TOTAL-PLEDGES', count, 120)

    self.response.headers['Content-Type'] = 'application/json'
    json.dump({'count':count}, self.response) 

def _set_memcache_value(self, key, type_, value):
    """Convert a string value and store the result in memcache.

    Args:
      key: String
      type_: String, describing what type the value should have in the cache.
      value: String, will be converted according to type_.

    Returns:
      Result of memcache.set(key, converted_value).  True if value was set.

    Raises:
      ValueError: Value can't be converted according to type_.
    """
    for _, converter, typestr in self.TYPES:
      if typestr == type_:
        value = converter(value)
        break
    else:
      raise ValueError('Type %s not supported.' % type_)
    return memcache.set(key, value) 

def _get_memcache_value_and_type(self, key):
    """Fetch value from memcache and detect its type.

    Args:
      key: String

    Returns:
      (value, type), value is a Python object or None if the key was not set in
      the cache, type is a string describing the type of the value.
    """
    try:
      value = memcache.get(key)
    except (pickle.UnpicklingError, AttributeError, EOFError, ImportError,
            IndexError), e:
      # Pickled data could be broken or the user might have stored custom class
      # instances in the cache (which can't be unpickled from here).
      msg = 'Failed to retrieve value from cache: %s' % e
      return msg, 'error' 

def update(cls, domain, time=0):
        """
        Updates the memcached stats for a given domain

        This is used when a report is updated so that memcached
        has the current stats.

        Args:
            domain - str - The domain to use for the namespace
            time - int - The timeout for stored keys (default: 5 seconds)
        """
        namespace = "{}|{}".format('stats', domain)

        for status in VALID_STATUSES:
            count = EmailReport.query(EmailReport.reported_domain == domain,
                                      EmailReport.status == status).count()
            memcache.set(
                key=status, namespace=namespace, value=count, time=time) 

def get_bot_version(host):
  """Retrieves the current bot version (SHA256) loaded on this server.

  The memcache is first checked for the version, otherwise the value
  is generated and then stored in the memcache.

  Returns:
    tuple(hash of the current bot version, dict of additional files).
  """
  signature = _get_signature(host)
  version = memcache.get('version-' + signature, namespace='bot_code')
  if version:
    return version, None

  # Need to calculate it.
  additionals = {'config/bot_config.py': get_bot_config().content}
  bot_dir = os.path.join(ROOT_DIR, 'swarming_bot')
  version = bot_archive.get_swarming_bot_version(
      bot_dir, host, utils.get_app_version(), additionals,
      local_config.settings())
  memcache.set('version-' + signature, version, namespace='bot_code', time=60)
  return version, additionals 

def update(cls, domain, time=0):
        """
        Updates the memcached stats for a given domain

        This is used when a report is updated so that memcached
        has the current stats.

        Args:
            domain - str - The domain to use for the namespace
            time - int - The timeout for stored keys (default: 5 seconds)
        """
        namespace = "{}|".format(domain)
        records = cls._get_from_datastore(domain, cls._memcache_result_count)
        memcache.set(
            key=cls._memcache_key,
            namespace=namespace,
            value=json.dumps(records),
            time=time) 

def _update_memcached(cls, domain, time=3600 * 24, records=None):
        """
        Updates memcached with the latest data from the datastore
        and returns that data. By default stores entries to expire after
        24 hours.
        """
        namespace = "{}|".format(domain)
        if not records:
            records = cls._get_from_datastore(domain,
                                              cls._memcache_date_offset)
        memcache.set(
            key=cls._memcache_key,
            namespace=namespace,
            value=json.dumps(records),
            time=time)
        return records 

def _pre_put_hook(self):
    super(TaskDimensions, self)._pre_put_hook()
    sets = set()
    for s in self.sets:
      s._pre_put_hook()
      sets.add('\000'.join(s.dimensions_flat))
    if len(sets) != len(self.sets):
      # Make sure there's no duplicate TaskDimensionsSet.
      raise datastore_errors.BadValueError(
          '%s.sets must all be unique' % self.__class__.__name__)


### Private APIs.


# Limit in rebuild_task_cache_async. Meant to be overridden in unit test. 

def assert_task_async(request):
  """Makes sure the TaskRequest dimensions, for each TaskProperties, are listed
  as a known queue.

  This function must be called before storing the TaskRequest in the DB.

  When a cache miss occurs, a task queue is triggered.

  Warning: the task will not be run until the task queue ran, which causes a
  user visible delay. There is no SLA but expected range is normally seconds at
  worst. This only occurs on new kind of requests, which is not that often in
  practice.
  """
  # It's important that the TaskRequest to not be stored in the DB yet, still
  # its key could be set.
  exp_ts = request.created_ts
  futures = []
  for i in range(request.num_task_slices):
    t = request.task_slice(i)
    exp_ts += datetime.timedelta(seconds=t.expiration_secs)
    futures.append(_assert_task_props_async(t.properties, exp_ts))
  for f in futures:
    yield f 

def get_task_queue_host():
  """Returns domain name of app engine instance to run a task queue task on.

  By default will use 'backend' module. Can be changed by calling
  set_task_queue_module during application startup.

  This domain name points to a matching version of appropriate app engine
  module - <version>.<module>.<app-id>.appspot.com where:
    version: version of the module that is calling this function.
    module: app engine module to execute task on.

  That way a task enqueued from version 'A' of default module would be executed
  on same version 'A' of backend module.
  """
  # modules.get_hostname sometimes fails with unknown internal error.
  # Cache its result in a memcache to avoid calling it too often.
  cache_key = 'task_queue_host:%s:%s' % (_task_queue_module, get_app_version())
  value = gae_memcache.get(cache_key)
  if not value:
    value = modules.get_hostname(module=_task_queue_module)
    gae_memcache.set(cache_key, value)
  return value 

def get_task_queue_host():
  """Returns domain name of app engine instance to run a task queue task on.

  By default will use 'backend' module. Can be changed by calling
  set_task_queue_module during application startup.

  This domain name points to a matching version of appropriate app engine
  module - <version>.<module>.<app-id>.appspot.com where:
    version: version of the module that is calling this function.
    module: app engine module to execute task on.

  That way a task enqueued from version 'A' of default module would be executed
  on same version 'A' of backend module.
  """
  # modules.get_hostname sometimes fails with unknown internal error.
  # Cache its result in a memcache to avoid calling it too often.
  cache_key = 'task_queue_host:%s:%s' % (_task_queue_module, get_app_version())
  value = gae_memcache.get(cache_key)
  if not value:
    value = modules.get_hostname(module=_task_queue_module)
    gae_memcache.set(cache_key, value)
  return value 

def GvizTableData(request):
    """Returns a string formatted for consumption by a Google Viz table."""
    #def throw_deadline():
    #    logging.info('MANUAL THROW!! DeadlineExceededError DeadlineExceededError')
    #    raise DeadlineExceededError
    #t = Timer(15.0, throw_deadline)

    test_set = None
    category = request.GET.get('category')
    if not category:
        return http.HttpResponseBadRequest('Must pass category=something')

    test_set = all_test_sets.GetTestSet(category)
    if not test_set:
        return http.HttpResponseBadRequest(
            'No test set was found for category=%s' % category)

    formatted_gviz_table_data = GetStats(request, test_set, 'gviz_table_data')
    return http.HttpResponse(formatted_gviz_table_data) 

def get_content(namespace, hash_key):
  """Returns the content from either memcache or datastore, when stored inline.

  This does NOT return data from GCS, it is up to the client to do that.

  Returns:
    tuple(content, ContentEntry)
    At most only one of the two is set.

  Raises LookupError if the content cannot be found.
  Raises ValueError if the hash_key is invalid.
  """
  memcache_entry = memcache.get(hash_key, namespace='table_%s' % namespace)
  if memcache_entry is not None:
    return (memcache_entry, None)
  else:
    # Raises ValueError
    key = get_entry_key(namespace, hash_key)
    entity = key.get()
    if entity is None:
      raise LookupError("namespace %s, key %s does not refer to anything" %
        (namespace, hash_key))
    return (entity.content, entity) 

def get_task_queue_host():
  """Returns domain name of app engine instance to run a task queue task on.

  By default will use 'backend' module. Can be changed by calling
  set_task_queue_module during application startup.

  This domain name points to a matching version of appropriate app engine
  module - <version>.<module>.<app-id>.appspot.com where:
    version: version of the module that is calling this function.
    module: app engine module to execute task on.

  That way a task enqueued from version 'A' of default module would be executed
  on same version 'A' of backend module.
  """
  # modules.get_hostname sometimes fails with unknown internal error.
  # Cache its result in a memcache to avoid calling it too often.
  cache_key = 'task_queue_host:%s:%s' % (_task_queue_module, get_app_version())
  value = gae_memcache.get(cache_key)
  if not value:
    value = modules.get_hostname(module=_task_queue_module)
    gae_memcache.set(cache_key, value)
  return value 

def add_values():
    # [START add_values]
    # Add a value if it doesn't exist in the cache
    # with a cache expiration of 1 hour.
    memcache.add(key="weather_USA_98105", value="raining", time=3600)

    # Set several values, overwriting any existing values for these keys.
    memcache.set_multi(
        {"USA_98115": "cloudy", "USA_94105": "foggy", "USA_94043": "sunny"},
        key_prefix="weather_",
        time=3600
    )

    # Atomically increment an integer value.
    memcache.set(key="counter", value=0)
    memcache.incr("counter")
    memcache.incr("counter")
    memcache.incr("counter")
    # [END add_values] 

def GetUserAccessToken(user_email, force_refresh=False):
  """Helper to get a refreshed access_token for a user via service account.

  Args:
    user_email: User email for which access_token will be retrieved.
    force_refresh: Boolean, if True force a token refresh.

  Returns:
    Cached access_token or a new one.
  """
  access_token = memcache.get(user_email, namespace=_CACHE_NAMESPACE)
  if access_token and not force_refresh:
    return access_token

  credentials = _GetSignedJwtAssertionCredentials(user_email)
  # Have observed the following error from refresh():
  # 'Unable to fetch URL: https://accounts.google.com/o/oauth2/token'
  _LOG.debug('Refreshing access token for %s.', user_email)
  credentials.refresh(http_utils.GetHttpObject())
  access_token = credentials.access_token
  if memcache.set(user_email, access_token, time=_ACCESS_TOKEN_CACHE_S,
                  namespace=_CACHE_NAMESPACE):
    return access_token
  raise recall_errors.MessageRecallCounterError(
      'Exceeded retry limit in GetUserAccessToken: %s.' % user_email) 

def import_config_set(config_set):
  """Imports a config set."""
  import_attempt_metric.increment(fields={'config_set': config_set})
  service_match = config.SERVICE_CONFIG_SET_RGX.match(config_set)
  if service_match:
    service_id = service_match.group(1)
    return import_service(service_id)

  project_match = config.PROJECT_CONFIG_SET_RGX.match(config_set)
  if project_match:
    project_id = project_match.group(1)
    return import_project(project_id)

  ref_match = config.REF_CONFIG_SET_RGX.match(config_set)
  if ref_match:
    project_id = ref_match.group(1)
    ref_name = ref_match.group(2)
    return import_ref(project_id, ref_name)

  raise ValueError('Invalid config set "%s' % config_set)


## A cron job that schedules an import push task for each config set 

def set(self, url, content):
    try:
      memcache.set(url, content, time=int(self._max_age), namespace=NAMESPACE)
    except Exception as e:
      logging.warning(e, exc_info=True) 

def get_module_version_list(module_list, tainted):
  """Returns a list of pairs (module name, version name) to fetch logs for.

  Arguments:
    module_list: list of modules to list, defaults to all modules.
    tainted: if False, excludes versions with '-tainted' in their name.
  """
  result = []
  if not module_list:
    # If the function it called too often, it'll raise a OverQuotaError. So
    # cache it for 10 minutes.
    module_list = gae_memcache.get('modules_list')
    if not module_list:
      module_list = modules.get_modules()
      gae_memcache.set('modules_list', module_list, time=10*60)

  for module in module_list:
    # If the function it called too often, it'll raise a OverQuotaError.
    # Versions is a bit more tricky since we'll loose data, since versions are
    # changed much more often than modules. So cache it for 1 minute.
    key = 'modules_list-' + module
    version_list = gae_memcache.get(key)
    if not version_list:
      version_list = modules.get_versions(module)
      gae_memcache.set(key, version_list, time=60)
    result.extend(
        (module, v) for v in version_list if tainted or '-tainted' not in v)
  return result


## Task queue 

def get(self):
    # Try to find a cached response for the current revision.
    auth_db_rev = model.get_auth_db_revision()
    cached_response = memcache.get(self.cache_key(auth_db_rev))
    if cached_response is not None:
      self.adjust_response_for_user(cached_response)
      self.send_response(cached_response)
      return

    # Grab a list of groups and corresponding revision for cache key.
    if not model.is_replica():
      def run():
        fut = model.AuthGroup.query(ancestor=model.root_key()).fetch_async()
        return model.get_auth_db_revision(), fut.get_result()
      auth_db_rev, group_list = ndb.transaction(run)
    else:
      auth_db = api.get_latest_auth_db()
      auth_db_rev = auth_db.auth_db_rev
      group_list = [auth_db.get_group(g) for g in auth_db.get_group_names()]

    # Currently AuthGroup entity contains a list of group members in the entity
    # body. It's an implementation detail that should not be relied upon.
    # Generally speaking, fetching a list of group members can be an expensive
    # operation, and group listing call shouldn't do it all the time. So throw
    # away all fields that enumerate group members.
    response = {
      'groups': [
        g.to_serializable_dict(
            with_id_as='name',
            exclude=('globs', 'members', 'nested'))
        for g in sorted(group_list, key=lambda x: x.key.string_id())
      ],
    }
    memcache.set(self.cache_key(auth_db_rev), response, time=24*3600)
    self.adjust_response_for_user(response)
    self.send_response(response) 

def _use_cached_or_fetch(cache_key, fetch_cb):
  """Implements caching layer for the public certificates.

  Caches certificate in both memcache and local instance memory. Uses
  probabilistic early expiration to avoid hitting the backend from multiple
  request handlers simultaneously when cache expires.

  'fetch_cb' is expected to return a dict to be passed to CertificateBundle
  constructor.
  """
  # Try local memory first.
  now = utils.time_time()
  with _certs_cache_lock:
    if cache_key in _certs_cache:
      certs, exp = _certs_cache[cache_key]
      if exp > now + 0.1 * _CERTS_CACHE_EXP_SEC * random.random():
        return certs

  # Try memcache now. Use same trick with random early expiration.
  entry = memcache.get(cache_key)
  if entry:
    certs_dict, exp = entry
    if exp > now + 0.1 * _CERTS_CACHE_EXP_SEC * random.random():
      certs = CertificateBundle(certs_dict)
      with _certs_cache_lock:
        _certs_cache[cache_key] = (certs, now + _CERTS_CACHE_EXP_SEC)
      return certs

  # Multiple concurrent fetches are possible, but it's not a big deal. The last
  # one wins.
  certs_dict = fetch_cb()
  exp = now + _CERTS_CACHE_EXP_SEC
  memcache.set(cache_key, (certs_dict, exp), time=exp)
  certs = CertificateBundle(certs_dict)
  with _certs_cache_lock:
    _certs_cache[cache_key] = (certs, exp)
  return certs 

def get(self):
    # Try to find a cached response for the current revision.
    auth_db_rev = model.get_auth_db_revision()
    cached_response = memcache.get(self.cache_key(auth_db_rev))
    if cached_response is not None:
      self.adjust_response_for_user(cached_response)
      self.send_response(cached_response)
      return

    # Grab a list of groups and corresponding revision for cache key.
    if not model.is_replica():
      def run():
        fut = model.AuthGroup.query(ancestor=model.root_key()).fetch_async()
        return model.get_auth_db_revision(), fut.get_result()
      auth_db_rev, group_list = ndb.transaction(run)
    else:
      auth_db = api.get_latest_auth_db()
      auth_db_rev = auth_db.auth_db_rev
      group_list = [auth_db.get_group(g) for g in auth_db.get_group_names()]

    # Currently AuthGroup entity contains a list of group members in the entity
    # body. It's an implementation detail that should not be relied upon.
    # Generally speaking, fetching a list of group members can be an expensive
    # operation, and group listing call shouldn't do it all the time. So throw
    # away all fields that enumerate group members.
    response = {
      'groups': [
        g.to_serializable_dict(
            with_id_as='name',
            exclude=('globs', 'members', 'nested'))
        for g in sorted(group_list, key=lambda x: x.key.string_id())
      ],
    }
    memcache.set(self.cache_key(auth_db_rev), response, time=24*3600)
    self.adjust_response_for_user(response)
    self.send_response(response) 

def _use_cached_or_fetch(cache_key, fetch_cb):
  """Implements caching layer for the public certificates.

  Caches certificate in both memcache and local instance memory. Uses
  probabilistic early expiration to avoid hitting the backend from multiple
  request handlers simultaneously when cache expires.

  'fetch_cb' is expected to return a dict to be passed to CertificateBundle
  constructor.
  """
  # Try local memory first.
  now = utils.time_time()
  with _certs_cache_lock:
    if cache_key in _certs_cache:
      certs, exp = _certs_cache[cache_key]
      if exp > now + 0.1 * _CERTS_CACHE_EXP_SEC * random.random():
        return certs

  # Try memcache now. Use same trick with random early expiration.
  entry = memcache.get(cache_key)
  if entry:
    certs_dict, exp = entry
    if exp > now + 0.1 * _CERTS_CACHE_EXP_SEC * random.random():
      certs = CertificateBundle(certs_dict)
      with _certs_cache_lock:
        _certs_cache[cache_key] = (certs, now + _CERTS_CACHE_EXP_SEC)
      return certs

  # Multiple concurrent fetches are possible, but it's not a big deal. The last
  # one wins.
  certs_dict = fetch_cb()
  exp = now + _CERTS_CACHE_EXP_SEC
  memcache.set(cache_key, (certs_dict, exp), time=exp)
  certs = CertificateBundle(certs_dict)
  with _certs_cache_lock:
    _certs_cache[cache_key] = (certs, exp)
  return certs 

def set(self, url, content):
    try:
      memcache.set(url, content, time=int(self._max_age), namespace=NAMESPACE)
    except Exception as e:
      logging.warning(e, exc_info=True) 

def expiration_jitter(now, expiration):
  """Returns expiration/next_tag pair to set in a ContentEntry."""
  jittered = random.uniform(1, 1.2) * expiration
  expiration = now + datetime.timedelta(seconds=jittered)
  next_tag = now + datetime.timedelta(seconds=jittered*0.1)
  return expiration, next_tag 

def set(self, url, content):
    try:
      memcache.set(url, content, time=int(self._max_age), namespace=NAMESPACE)
    except Exception as e:
      LOGGER.warning(e, exc_info=True) 

def get(self):
        # [START sharing]
        self.response.headers['Content-Type'] = 'text/plain'

        who = memcache.get('who')
        self.response.write('Previously incremented by %s\n' % who)
        memcache.set('who', 'Python')

        count = memcache.incr('count', 1, initial_value=0)
        self.response.write('Count incremented by Python = %s\n' % count)
        # [END sharing] 

def get(self):
        value = 3
        # [START memcache-failure]
        if not memcache.set('counter', value):
            logging.error("Memcache set failed")
            # Other error handling here
        # [END memcache-failure]
        self.response.content_type = 'text/html'
        self.response.write('done') 

def get(self):
        key = "some key"
        seconds = 5
        memcache.set(key, "some value")
        # [START memcache-delete]
        memcache.delete(key, seconds)  # clears cache
        # write to persistent datastore
        # Do not attempt to put new value in cache, first reader will do that
        # [END memcache-delete]
        self.response.content_type = 'text/html'
        self.response.write('done')