Python google.appengine.ext.ndb.StructuredProperty() Examples

def get_allowed_properties(self, model_cls):
    """Returns a set of property names to consider when converting a dictionary.

    When working with StructuredProperty based on regular ndb.Model, export all
    fields. Otherwise use model_cls.serializable_properties and
    self.field_mode_predicate to figure out set of properties to use.

    Return value of None means all defined properties should be used.
    """
    assert issubclass(model_cls, ndb.Model)
    assert not issubclass(model_cls, ndb.Expando), 'Expando is not supported'
    if not issubclass(model_cls, SerializableModelMixin):
      return None
    if model_cls.serializable_properties is None:
      return None
    return set(
        field for field, mode in model_cls.serializable_properties.items()
        if self.field_mode_predicate(mode)) 

def _test_repeated_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    class Inner(ndb.Model):
      a = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      inner = structured_cls(Inner, repeated=True)

    # Repeated structured property -> list of dicts.
    entity = Outter()
    entity.inner.extend([Inner(a=1), Inner(a=2)])
    self.assertEqual(
        {'inner': [{'a': 1}, {'a': 2}]},
        entity.to_serializable_dict())

    # Reverse also works.
    self.assertEqual(
        entity,
        Outter.from_serializable_dict({'inner': [{'a': 1}, {'a': 2}]})) 

def get_allowed_properties(self, model_cls):
    """Returns a set of property names to consider when converting a dictionary.

    When working with StructuredProperty based on regular ndb.Model, export all
    fields. Otherwise use model_cls.serializable_properties and
    self.field_mode_predicate to figure out set of properties to use.

    Return value of None means all defined properties should be used.
    """
    assert issubclass(model_cls, ndb.Model)
    assert not issubclass(model_cls, ndb.Expando), 'Expando is not supported'
    if not issubclass(model_cls, SerializableModelMixin):
      return None
    if model_cls.serializable_properties is None:
      return None
    return set(
        field for field, mode in model_cls.serializable_properties.items()
        if self.field_mode_predicate(mode)) 

def _test_repeated_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    class Inner(ndb.Model):
      a = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      inner = structured_cls(Inner, repeated=True)

    # Repeated structured property -> list of dicts.
    entity = Outter()
    entity.inner.extend([Inner(a=1), Inner(a=2)])
    self.assertEqual(
        {'inner': [{'a': 1}, {'a': 2}]},
        entity.to_serializable_dict())

    # Reverse also works.
    self.assertEqual(
        entity,
        Outter.from_serializable_dict({'inner': [{'a': 1}, {'a': 2}]})) 

def _test_repeated_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    class Inner(ndb.Model):
      a = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      inner = structured_cls(Inner, repeated=True)

    # Repeated structured property -> list of dicts.
    entity = Outter()
    entity.inner.extend([Inner(a=1), Inner(a=2)])
    self.assertEqual(
        {'inner': [{'a': 1}, {'a': 2}]},
        entity.to_serializable_dict())

    # Reverse also works.
    self.assertEqual(
        entity,
        Outter.from_serializable_dict({'inner': [{'a': 1}, {'a': 2}]})) 

def get_allowed_properties(self, model_cls):
    """Returns a set of property names to consider when converting a dictionary.

    When working with StructuredProperty based on regular ndb.Model, export all
    fields. Otherwise use model_cls.serializable_properties and
    self.field_mode_predicate to figure out set of properties to use.

    Return value of None means all defined properties should be used.
    """
    assert issubclass(model_cls, ndb.Model)
    assert not issubclass(model_cls, ndb.Expando), 'Expando is not supported'
    if not issubclass(model_cls, SerializableModelMixin):
      return None
    if model_cls.serializable_properties is None:
      return None
    return set(
        field for field, mode in model_cls.serializable_properties.items()
        if self.field_mode_predicate(mode)) 

def get_allowed_properties(self, model_cls):
    """Returns a set of property names to consider when converting a dictionary.

    When working with StructuredProperty based on regular ndb.Model, export all
    fields. Otherwise use model_cls.serializable_properties and
    self.field_mode_predicate to figure out set of properties to use.

    Return value of None means all defined properties should be used.
    """
    assert issubclass(model_cls, ndb.Model)
    assert not issubclass(model_cls, ndb.Expando), 'Expando is not supported'
    if not issubclass(model_cls, SerializableModelMixin):
      return None
    if model_cls.serializable_properties is None:
      return None
    return set(
        field for field, mode in model_cls.serializable_properties.items()
        if self.field_mode_predicate(mode)) 

def _test_repeated_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    class Inner(ndb.Model):
      a = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      inner = structured_cls(Inner, repeated=True)

    # Repeated structured property -> list of dicts.
    entity = Outter()
    entity.inner.extend([Inner(a=1), Inner(a=2)])
    self.assertEqual(
        {'inner': [{'a': 1}, {'a': 2}]},
        entity.to_serializable_dict())

    # Reverse also works.
    self.assertEqual(
        entity,
        Outter.from_serializable_dict({'inner': [{'a': 1}, {'a': 2}]})) 

def _test_repeated_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    class Inner(ndb.Model):
      a = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      inner = structured_cls(Inner, repeated=True)

    # Repeated structured property -> list of dicts.
    entity = Outter()
    entity.inner.extend([Inner(a=1), Inner(a=2)])
    self.assertEqual(
        {'inner': [{'a': 1}, {'a': 2}]},
        entity.to_serializable_dict())

    # Reverse also works.
    self.assertEqual(
        entity,
        Outter.from_serializable_dict({'inner': [{'a': 1}, {'a': 2}]})) 

def get_allowed_properties(self, model_cls):
    """Returns a set of property names to consider when converting a dictionary.

    When working with StructuredProperty based on regular ndb.Model, export all
    fields. Otherwise use model_cls.serializable_properties and
    self.field_mode_predicate to figure out set of properties to use.

    Return value of None means all defined properties should be used.
    """
    assert issubclass(model_cls, ndb.Model)
    assert not issubclass(model_cls, ndb.Expando), 'Expando is not supported'
    if not issubclass(model_cls, SerializableModelMixin):
      return None
    if model_cls.serializable_properties is None:
      return None
    return set(
        field for field, mode in model_cls.serializable_properties.items()
        if self.field_mode_predicate(mode)) 

def _test_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    # Plain ndb.Model.
    class InnerSimple(ndb.Model):
      a = ndb.IntegerProperty()

    # With SerializableModelMixin.
    class InnerSmart(ndb.Model, serializable.SerializableModelMixin):
      serializable_properties = {
        'a': serializable.READABLE | serializable.WRITABLE,
      }
      a = ndb.IntegerProperty()
      b = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      simple = structured_cls(InnerSimple)
      smart = structured_cls(InnerSmart)

    # InnerSimple gets serialized entirely, while only readable fields
    # on InnerSmart are serialized.
    entity = Outter()
    entity.simple = InnerSimple(a=1)
    entity.smart = InnerSmart(a=2, b=3)
    self.assertEqual(
        {'simple': {'a': 1}, 'smart': {'a': 2}},
        entity.to_serializable_dict())

    # Works backwards as well. Note that 'convert_serializable_dict' returns
    # a dictionary that can be fed to entity's 'populate' or constructor. Entity
    # by itself is smart enough to transform subdicts into structured
    # properties.
    self.assertEqual(
        Outter(simple=InnerSimple(a=1), smart=InnerSmart(a=2)),
        Outter.from_serializable_dict({'simple': {'a': 1}, 'smart': {'a': 2}})) 

def test_repeated_structured_properties(self):
    """Test handling of StructuredProperty(repeated=True)."""
    self._test_repeated_structured_properties_class(ndb.StructuredProperty) 

def test_structured_properties(self):
    """Test handling of StructuredProperty."""
    self._test_structured_properties_class(ndb.StructuredProperty) 

def _test_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    # Plain ndb.Model.
    class InnerSimple(ndb.Model):
      a = ndb.IntegerProperty()

    # With SerializableModelMixin.
    class InnerSmart(ndb.Model, serializable.SerializableModelMixin):
      serializable_properties = {
        'a': serializable.READABLE | serializable.WRITABLE,
      }
      a = ndb.IntegerProperty()
      b = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      simple = structured_cls(InnerSimple)
      smart = structured_cls(InnerSmart)

    # InnerSimple gets serialized entirely, while only readable fields
    # on InnerSmart are serialized.
    entity = Outter()
    entity.simple = InnerSimple(a=1)
    entity.smart = InnerSmart(a=2, b=3)
    self.assertEqual(
        {'simple': {'a': 1}, 'smart': {'a': 2}},
        entity.to_serializable_dict())

    # Works backwards as well. Note that 'convert_serializable_dict' returns
    # a dictionary that can be fed to entity's 'populate' or constructor. Entity
    # by itself is smart enough to transform subdicts into structured
    # properties.
    self.assertEqual(
        Outter(simple=InnerSimple(a=1), smart=InnerSmart(a=2)),
        Outter.from_serializable_dict({'simple': {'a': 1}, 'smart': {'a': 2}})) 

def test_repeated_structured_properties(self):
    """Test handling of StructuredProperty(repeated=True)."""
    self._test_repeated_structured_properties_class(ndb.StructuredProperty) 

def test_structured_properties(self):
    """Test handling of StructuredProperty."""
    self._test_structured_properties_class(ndb.StructuredProperty) 

def _test_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    # Plain ndb.Model.
    class InnerSimple(ndb.Model):
      a = ndb.IntegerProperty()

    # With SerializableModelMixin.
    class InnerSmart(ndb.Model, serializable.SerializableModelMixin):
      serializable_properties = {
        'a': serializable.READABLE | serializable.WRITABLE,
      }
      a = ndb.IntegerProperty()
      b = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      simple = structured_cls(InnerSimple)
      smart = structured_cls(InnerSmart)

    # InnerSimple gets serialized entirely, while only readable fields
    # on InnerSmart are serialized.
    entity = Outter()
    entity.simple = InnerSimple(a=1)
    entity.smart = InnerSmart(a=2, b=3)
    self.assertEqual(
        {'simple': {'a': 1}, 'smart': {'a': 2}},
        entity.to_serializable_dict())

    # Works backwards as well. Note that 'convert_serializable_dict' returns
    # a dictionary that can be fed to entity's 'populate' or constructor. Entity
    # by itself is smart enough to transform subdicts into structured
    # properties.
    self.assertEqual(
        Outter(simple=InnerSimple(a=1), smart=InnerSmart(a=2)),
        Outter.from_serializable_dict({'simple': {'a': 1}, 'smart': {'a': 2}})) 

def test_repeated_structured_properties(self):
    """Test handling of StructuredProperty(repeated=True)."""
    self._test_repeated_structured_properties_class(ndb.StructuredProperty) 

def test_structured_properties(self):
    """Test handling of StructuredProperty."""
    self._test_structured_properties_class(ndb.StructuredProperty) 

def _test_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    # Plain ndb.Model.
    class InnerSimple(ndb.Model):
      a = ndb.IntegerProperty()

    # With SerializableModelMixin.
    class InnerSmart(ndb.Model, serializable.SerializableModelMixin):
      serializable_properties = {
        'a': serializable.READABLE | serializable.WRITABLE,
      }
      a = ndb.IntegerProperty()
      b = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      simple = structured_cls(InnerSimple)
      smart = structured_cls(InnerSmart)

    # InnerSimple gets serialized entirely, while only readable fields
    # on InnerSmart are serialized.
    entity = Outter()
    entity.simple = InnerSimple(a=1)
    entity.smart = InnerSmart(a=2, b=3)
    self.assertEqual(
        {'simple': {'a': 1}, 'smart': {'a': 2}},
        entity.to_serializable_dict())

    # Works backwards as well. Note that 'convert_serializable_dict' returns
    # a dictionary that can be fed to entity's 'populate' or constructor. Entity
    # by itself is smart enough to transform subdicts into structured
    # properties.
    self.assertEqual(
        Outter(simple=InnerSimple(a=1), smart=InnerSmart(a=2)),
        Outter.from_serializable_dict({'simple': {'a': 1}, 'smart': {'a': 2}})) 

def test_repeated_structured_properties(self):
    """Test handling of StructuredProperty(repeated=True)."""
    self._test_repeated_structured_properties_class(ndb.StructuredProperty) 

def test_structured_properties(self):
    """Test handling of StructuredProperty."""
    self._test_structured_properties_class(ndb.StructuredProperty) 

def _test_structured_properties_class(self, structured_cls):
    """Common testing for StructuredProperty and LocalStructuredProperty."""
    # Plain ndb.Model.
    class InnerSimple(ndb.Model):
      a = ndb.IntegerProperty()

    # With SerializableModelMixin.
    class InnerSmart(ndb.Model, serializable.SerializableModelMixin):
      serializable_properties = {
        'a': serializable.READABLE | serializable.WRITABLE,
      }
      a = ndb.IntegerProperty()
      b = ndb.IntegerProperty()

    class Outter(ndb.Model, serializable.SerializableModelMixin):
      simple = structured_cls(InnerSimple)
      smart = structured_cls(InnerSmart)

    # InnerSimple gets serialized entirely, while only readable fields
    # on InnerSmart are serialized.
    entity = Outter()
    entity.simple = InnerSimple(a=1)
    entity.smart = InnerSmart(a=2, b=3)
    self.assertEqual(
        {'simple': {'a': 1}, 'smart': {'a': 2}},
        entity.to_serializable_dict())

    # Works backwards as well. Note that 'convert_serializable_dict' returns
    # a dictionary that can be fed to entity's 'populate' or constructor. Entity
    # by itself is smart enough to transform subdicts into structured
    # properties.
    self.assertEqual(
        Outter(simple=InnerSimple(a=1), smart=InnerSmart(a=2)),
        Outter.from_serializable_dict({'simple': {'a': 1}, 'smart': {'a': 2}})) 

def test_repeated_structured_properties(self):
    """Test handling of StructuredProperty(repeated=True)."""
    self._test_repeated_structured_properties_class(ndb.StructuredProperty) 

def test_structured_properties(self):
    """Test handling of StructuredProperty."""
    self._test_structured_properties_class(ndb.StructuredProperty) 

def test_generate_entity_schema(self):

    class NestedTestModel(ndb.Model):
      nested_string_attribute = ndb.StringProperty()

    class TestModel(ndb.Model):
      string_attribute = ndb.StringProperty()
      integer_attribute = ndb.IntegerProperty()
      boolean_attribute = ndb.BooleanProperty()
      nested_attribute = ndb.StructuredProperty(NestedTestModel)

    schema = bigquery._generate_entity_schema(TestModel())
    expected_schema_names = _populate_schema_names(self.entity_schema)
    schema_names = _populate_schema_names(schema)
    self.assertCountEqual(expected_schema_names, schema_names) 

def _build_model_from_data(self, data, cls, model=None):
        """Builds a model instance (according to `cls`) from user input and returns it. Updates an existing model instance if given.
        Raises exceptions if input data is invalid."""

        # Translate the property names (this is done before the filtering in order to get the original property names by which the filtering is done)
        data = translate_property_names(data, cls, 'input')

        # Transform any raw input data into appropriate NDB properties - write all transformed properties
        # into another dict (so any other unauthorized properties will be ignored).
        input_properties = { }
        for (name, prop) in cls._properties.iteritems():
            if name not in data: continue # Input not given by user

            if prop._repeated:
                # This property is repeated (i.e. an array of values)
                input_properties[name] = [self._value_to_property(value, prop) for value in data[name]]
            else:
                input_properties[name] = self._value_to_property(data[name], prop)

        if not model and getattr(cls, 'RESTMeta', None) and getattr(cls.RESTMeta, 'use_input_id', False):
            if 'id' not in data:
                raise RESTException('id field is required')
            input_properties['id'] = data['id']

        # Filter the input properties
        included_properties = get_included_properties(cls, 'input')
        input_properties = dict((k,v) for k,v in input_properties.iteritems() if k in included_properties)

        # Set the user owner property to the currently logged-in user (if it's defined for the model class) - note that we're doing this check on the input `cls` parameter
        # and not the self.model class, since we need to support when a model has an inner StructuredProperty, and that model has its own RESTMeta definition.
        if hasattr(cls, 'RESTMeta') and hasattr(cls.RESTMeta, 'user_owner_property'):
            if not model and self.user:
                # Only perform this update when creating a new model - otherwise, each update might change this (very problematic in case an
                # admin updates another user's model instance - it'll change model ownership from that user to the admin)
                input_properties[cls.RESTMeta.user_owner_property] = self.user.key

        if not model:
            # Create a new model instance
            model = cls(**input_properties)
        else:
            # Update an existing model instance
            model.populate(**input_properties)

        return model 

def _value_to_property(self, value, prop):
        """Converts raw data value into an appropriate NDB property"""
        if isinstance(prop, ndb.KeyProperty):
            if value is None:
                return None
            try:
                return ndb.Key(urlsafe=value)
            except ProtocolBufferDecodeError as e:
                if prop._kind is not None:
                    model_class = ndb.Model._kind_map.get(prop._kind)
                    if getattr(model_class, 'RESTMeta', None) and getattr(model_class.RESTMeta, 'use_input_id', False):
                        return ndb.Key(model_class, value)
            raise RESTException('invalid key: {}'.format(value) )
        elif isinstance(prop, ndb.TimeProperty):
            if dateutil is None:
                try:
                    return datetime.strptime(value, "%H:%M:%S").time()
                except ValueError as e:
                    raise RESTException("Invalid time. Must be in ISO 8601 format.")
            else:
                return dateutil.parser.parse(value).time()
        elif  isinstance(prop, ndb.DateProperty):
            if dateutil is None:
                try:
                    return datetime.strptime(value, "%Y-%m-%d").date()
                except ValueError as e:
                    raise RESTException("Invalid date. Must be in ISO 8601 format.")
            else:
                return dateutil.parser.parse(value).date()
        elif isinstance(prop, ndb.DateTimeProperty):
            if dateutil is None:
                try:
                    return datetime.strptime(value, "%Y-%m-%dT%H:%M:%S")
                except ValueError as e:
                    raise RESTException("Invalid datetime. Must be in ISO 8601 format.")
            else:
                return dateutil.parser.parse(value)
        elif isinstance(prop, ndb.GeoPtProperty):
            # Convert from string (formatted as '52.37, 4.88') to GeoPt
            return ndb.GeoPt(value)
        elif isinstance(prop, ndb.StructuredProperty):
            # It's a structured property - the input data is a dict - recursively parse it as well
            return self._build_model_from_data(value, prop._modelclass)
        else:
            # Return as-is (no need for further manipulation)
            return value