com.google.datastore.v1.Value Java Examples

/**
 * Grabs the schema for what data is in the Entity.
 * @param entity a populated entity
 * @return a schema of what kind of data is in the entity
 */
private JsonObject entitySchema(Entity entity) {
  JsonObject jsonObject = new JsonObject();
  entity.getPropertiesMap().entrySet().stream().forEach(entrySet -> {
    String key = entrySet.getKey();
    Value value = entrySet.getValue();
    switch (value.getValueTypeCase()) {
      case ENTITY_VALUE:
        jsonObject.add(key, entitySchema(value.getEntityValue()));
        break;
      case ARRAY_VALUE:
        jsonObject.add(key, arraySchema(value.getArrayValue()));
        break;
      default:
        jsonObject.addProperty(key, value.getValueTypeCase().toString());
    }
  });
  return jsonObject;
}
 

/**
 * Tests that BigQueryConverters.columnToValue() returns a String {@link Value} when the BigQuery
 * column is a RECORD.
 */
@Test
public void testColumnToValueRecord() {
  TableFieldSchema column = generateNestedTableFieldSchema();
  Record record = generateNestedAvroRecord();
  boolean isThrown = false;
  String message = null;
  try {
    Value value = BigQueryConverters.columnToValue(column, record.get("address"));
  } catch (IllegalArgumentException e) {
    isThrown = true;
    message = e.getMessage();
  }
  assertTrue(isThrown);
  assertTrue(message != null);
  assertEquals(message, "Column [address] of type [RECORD] not supported.");
}
 

@ProcessElement
public void processElement(ProcessContext c) {
  Entity.Builder entityBuilder = Entity.newBuilder();
  com.google.datastore.v1.Key key =
      makeKey(makeKey(kind, ancestorKey).build(), kind, c.element().getKey()).build();

  entityBuilder.setKey(key);
  List<Value> candidates = new ArrayList<>();
  Map<String, Value> properties = new HashMap<>();
  for (CompletionCandidate tag : c.element().getValue()) {
    Entity.Builder tagEntity = Entity.newBuilder();
    properties.put("tag", makeValue(tag.value).build());
    properties.put("count", makeValue(tag.count).build());
    candidates.add(makeValue(tagEntity).build());
  }
  properties.put("candidates", makeValue(candidates).build());
  entityBuilder.putAllProperties(properties);
  c.output(entityBuilder.build());
}
 

/**
 * Tests that BigQueryConverters.columnToValue() throws IllegalArgumentException when the BigQuery
 * column is an invalid TIMESTAMP.
 */
@Test
public void testColumnToValueTimestampInvalid() {
  TableFieldSchema column =
      new TableFieldSchema().setName(invalidTimestampField).setType("TIMESTAMP");
  Record record =
      generateSingleFieldAvroRecord(
          invalidTimestampField,
          "long",
          invalidTimestampFieldDesc,
          invalidTimestampFieldValueNanos);
  boolean isThrown = false;
  try {
    Value value = BigQueryConverters.columnToValue(column, record.get(invalidTimestampField));
  } catch (IllegalArgumentException e) {
    isThrown = true;
  }
  assertTrue(isThrown);
}
 

/**
 * List the greetings in the specified guestbook.
 */
private void listGreetings(String guestbookName) throws DatastoreException {
  Query.Builder query = Query.newBuilder();
  query.addKindBuilder().setName(GREETING_KIND);
  query.setFilter(makeFilter(KEY_PROPERTY, PropertyFilter.Operator.HAS_ANCESTOR,
      makeValue(makeKey(GUESTBOOK_KIND, guestbookName))));
  query.addOrder(makeOrder(DATE_PROPERTY, PropertyOrder.Direction.DESCENDING));

  List<Entity> greetings = runQuery(query.build());
  if (greetings.isEmpty()) {
    System.out.println("no greetings in " + guestbookName);
  }
  for (Entity greeting : greetings) {
    Map<String, Value> propertyMap = greeting.getPropertiesMap();
    System.out.println(
        DatastoreHelper.toDate(propertyMap.get(DATE_PROPERTY)) + ": " +
        DatastoreHelper.getString(propertyMap.get(USER_PROPERTY)) + " says " +
        DatastoreHelper.getString(propertyMap.get(MESSAGE_PROPERTY)));
  }
}
 

/**
 * @return the boolean contained in value
 * @throws IllegalArgumentException if the value does not contain a boolean.
 */
public static boolean getBoolean(Value value) {
  if (value.getValueTypeCase() != ValueTypeCase.BOOLEAN_VALUE) {
    throw new IllegalArgumentException("Value does not contain a boolean.");
  }
  return value.getBooleanValue();
}
 

/**
 * @return the string contained in value
 * @throws IllegalArgumentException if the value does not contain a string.
 */
public static String getString(Value value) {
  if (value.getValueTypeCase() != ValueTypeCase.STRING_VALUE) {
    throw new IllegalArgumentException("Value does not contain a string.");
  }
  return value.getStringValue();
}
 

/**
 * @return the entity contained in value
 * @throws IllegalArgumentException if the value does not contain an entity.
 */
public static Entity getEntity(Value value) {
  if (value.getValueTypeCase() != ValueTypeCase.ENTITY_VALUE) {
    throw new IllegalArgumentException("Value does not contain an Entity.");
  }
  return value.getEntityValue();
}
 

/**
 * @return the blob contained in value
 * @throws IllegalArgumentException if the value does not contain a blob.
 */
public static ByteString getByteString(Value value) {
  if (value.getMeaning() == 18 && value.getValueTypeCase() == ValueTypeCase.STRING_VALUE) {
    return value.getStringValueBytes();
  } else if (value.getValueTypeCase() == ValueTypeCase.BLOB_VALUE) {
    return value.getBlobValue();
  }
  throw new IllegalArgumentException("Value does not contain a blob.");
}
 

/**
 * @return the key contained in value
 * @throws IllegalArgumentException if the value does not contain a key.
 */
public static Key getKey(Value value) {
  if (value.getValueTypeCase() != ValueTypeCase.KEY_VALUE) {
    throw new IllegalArgumentException("Value does not contain a key.");
  }
  return value.getKeyValue();
}
 

/**
 * @return the double contained in value
 * @throws IllegalArgumentException if the value does not contain a double.
 */
public static double getDouble(Value value) {
  if (value.getValueTypeCase() != ValueTypeCase.DOUBLE_VALUE) {
    throw new IllegalArgumentException("Value does not contain a double.");
  }
  return value.getDoubleValue();
}
 

/**
 * Make an array value containing the specified values.
 */
public static Value.Builder makeValue(Value.Builder value1, Value.Builder value2,
    Value.Builder... rest) {
  ArrayValue.Builder arrayValue = ArrayValue.newBuilder();
  arrayValue.addValues(value1);
  arrayValue.addValues(value2);
  for (Value.Builder builder : rest) {
    arrayValue.addValues(builder);
  }
  return Value.newBuilder().setArrayValue(arrayValue);
}
 

/**
 * Make a list value containing the specified values.
 */
public static Value.Builder makeValue(Value value1, Value value2, Value... rest) {
  ArrayValue.Builder arrayValue = ArrayValue.newBuilder();
  arrayValue.addValues(value1);
  arrayValue.addValues(value2);
  arrayValue.addAllValues(Arrays.asList(rest));
  return Value.newBuilder().setArrayValue(arrayValue);
}
 

/**
 * @return the long contained in value
 * @throws IllegalArgumentException if the value does not contain a long.
 */
public static long getLong(Value value) {
  if (value.getValueTypeCase() != ValueTypeCase.INTEGER_VALUE) {
    throw new IllegalArgumentException("Value does not contain an integer.");
  }
  return value.getIntegerValue();
}
 

/**
 * Converts a {@code Row} value to an appropriate DataStore {@code Value} object.
 *
 * @param value {@code Row} value to convert.
 * @throws IllegalStateException when no mapping function for object of given type exists.
 * @return resulting {@code Value}.
 */
private Value mapObjectToValue(Object value) {
  if (value == null) {
    return Value.newBuilder().build();
  }

  if (Boolean.class.equals(value.getClass())) {
    return makeValue((Boolean) value).build();
  } else if (Byte.class.equals(value.getClass())) {
    return makeValue((Byte) value).build();
  } else if (Long.class.equals(value.getClass())) {
    return makeValue((Long) value).build();
  } else if (Short.class.equals(value.getClass())) {
    return makeValue((Short) value).build();
  } else if (Integer.class.equals(value.getClass())) {
    return makeValue((Integer) value).build();
  } else if (Double.class.equals(value.getClass())) {
    return makeValue((Double) value).build();
  } else if (Float.class.equals(value.getClass())) {
    return makeValue((Float) value).build();
  } else if (String.class.equals(value.getClass())) {
    return makeValue((String) value).build();
  } else if (Instant.class.equals(value.getClass())) {
    return makeValue(((Instant) value).toDate()).build();
  } else if (byte[].class.equals(value.getClass())) {
    return makeValue(ByteString.copyFrom((byte[]) value)).build();
  } else if (value instanceof Row) {
    // Recursive conversion to handle nested rows.
    Row row = (Row) value;
    return makeValue(constructEntityFromRow(row.getSchema(), row)).build();
  } else if (value instanceof Collection) {
    // Recursive to handle nested collections.
    Collection<Object> collection = (Collection<Object>) value;
    List<Value> arrayValues =
        collection.stream().map(this::mapObjectToValue).collect(Collectors.toList());
    return makeValue(arrayValues).build();
  }
  throw new IllegalStateException(
      "No conversion exists from type: " + value.getClass() + " to DataStove Value.");
}
 

/**
 * Converts an entire {@code Row} to an appropriate DataStore {@code Entity.Builder}.
 *
 * @param row {@code Row} to convert.
 * @return resulting {@code Entity.Builder}.
 */
private Entity.Builder constructEntityFromRow(Schema schema, Row row) {
  Entity.Builder entityBuilder = Entity.newBuilder();
  for (Schema.Field field : schema.getFields()) {
    Value val = mapObjectToValue(row.getValue(field.getName()));
    entityBuilder.putProperties(field.getName(), val);
  }
  return entityBuilder;
}
 

/**
 * @return the timestamp in microseconds contained in value
 * @throws IllegalArgumentException if the value does not contain a timestamp.
 */
public static long getTimestamp(Value value) {
  if (value.getMeaning() == 18 && value.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE) {
    return value.getIntegerValue();
  } else if (value.getValueTypeCase() == ValueTypeCase.TIMESTAMP_VALUE) {
    return toMicroseconds(value.getTimestampValue());
  }
  throw new IllegalArgumentException("Value does not contain a timestamp.");
}
 

/**
 * Converts all properties of an {@code Entity} to Beam {@code Row}.
 *
 * @param schema Target row {@code Schema}.
 * @param values A map of property names and values.
 * @return resulting Beam {@code Row}.
 */
private Row extractRowFromProperties(Schema schema, Map<String, Value> values) {
  Row.Builder builder = Row.withSchema(schema);
  // It is not a guarantee that the values will be in the same order as the schema.
  // Maybe metadata:
  // https://cloud.google.com/appengine/docs/standard/python/datastore/metadataqueries
  // TODO: figure out in what order the elements are in (without relying on Beam schema).
  for (Schema.Field field : schema.getFields()) {
    Value val = values.get(field.getName());
    builder.addValue(convertValueToObject(field.getType(), val));
  }
  return builder.build();
}
 

@DoFn.ProcessElement
public void processElement(ProcessContext context) {
  Entity entity = context.element();
  ImmutableMap.Builder<String, Value> mapBuilder = ImmutableMap.builder();
  mapBuilder.put(keyField, makeValue(entity.getKey()).build());
  mapBuilder.putAll(entity.getPropertiesMap());

  context.output(extractRowFromProperties(schema, mapBuilder.build()));
}
 

/**
 * @return the array contained in value as a list.
 * @throws IllegalArgumentException if the value does not contain an array.
 */
public static List<Value> getList(Value value) {
  if (value.getValueTypeCase() != ValueTypeCase.ARRAY_VALUE) {
    throw new IllegalArgumentException("Value does not contain an array.");
  }
  return value.getArrayValue().getValuesList();
}
 

/**
 * Make a filter on a property for use in a query.
 */
public static Filter.Builder makeFilter(String property, PropertyFilter.Operator operator,
    Value value) {
  return Filter.newBuilder()
      .setPropertyFilter(PropertyFilter.newBuilder()
          .setProperty(makePropertyReference(property))
          .setOp(operator)
          .setValue(value));
}
 

/**
 * Tests that BigQueryConverters.columnToValue() returns a String {@link Value} when the BigQuery
 * column is a TIME.
 */
@Test
public void testColumnToValueTime() {
  TableFieldSchema column = new TableFieldSchema().setName(timeField).setType("TIME");
  Record record =
      generateSingleFieldAvroRecord(timeField, "string", timeFieldDesc, timeFieldValue);
  Value value = BigQueryConverters.columnToValue(column, record.get(timeField));
  assertEquals(timeFieldValue, value.getStringValue());
}
 

/**
 * Tests that BigQueryConverters.columnToValue() returns an Integer {@link Value} when the
 * BigQuery column is an INT64.
 */
@Test
public void testColumnToValueInt64() {
  TableFieldSchema column = new TableFieldSchema().setName(int64Field).setType("INT64");
  Record record =
      generateSingleFieldAvroRecord(int64Field, "int", int64FieldDesc, int64FieldValue);
  Value value = BigQueryConverters.columnToValue(column, record.get(int64Field));
  assertEquals(int64FieldValue, value.getIntegerValue());
}
 

/**
 * Tests that BigQueryConverters.columnToValue() returns an Integer {@link Value} when the
 * BigQuery column is an INTEGER.
 */
@Test
public void testColumnToValueInteger() {
  TableFieldSchema column = new TableFieldSchema().setName(integerField).setType("INTEGER");
  Record record =
      generateSingleFieldAvroRecord(integerField, "int", integerFieldDesc, integerFieldValue);
  Value value = BigQueryConverters.columnToValue(column, record.get(integerField));
  assertEquals(integerFieldValue, value.getIntegerValue());
}
 

/**
 * Tests that BigQueryConverters.columnToValue() returns a non-indexed String {@link Value} when
 * the BigQuery column is a STRING longer than 1500 bytes.
 */
@Test
public void testColumnToValueStringLong() {
  TableFieldSchema column = new TableFieldSchema().setName(longStringField).setType("STRING");
  Record record =
      generateSingleFieldAvroRecord(
          longStringField, "string", longStringFieldDesc, longStringFieldValue);
  Value value = BigQueryConverters.columnToValue(column, record.get(longStringField));
  assertEquals(longStringFieldValue, value.getStringValue());
  assertTrue(value.getExcludeFromIndexes());
}
 

/**
 * Tests that BigQueryConverters.columnToValue() returns an indexed String {@link Value} when the
 * BigQuery column is a STRING of less than 1500 bytes.
 */
@Test
public void testColumnToValueStringShort() {
  TableFieldSchema column = new TableFieldSchema().setName(shortStringField).setType("STRING");
  Record record =
      generateSingleFieldAvroRecord(
          shortStringField, "string", shortStringFieldDesc, shortStringFieldValue);
  Value value = BigQueryConverters.columnToValue(column, record.get(shortStringField));
  assertEquals(shortStringFieldValue, value.getStringValue());
  assertFalse(value.getExcludeFromIndexes());
}
 

/**
 * Tests that BigQueryConverters.columnToValue() returns a null {@link Value} when the BigQuery
 * column is null.
 */
@Test
public void testColumnToValueNull() {
  TableFieldSchema column = new TableFieldSchema().setName(nullField).setType("STRING");
  Record record = generateSingleFieldAvroRecord(nullField, "null", nullFieldDesc, null);
  Value value = BigQueryConverters.columnToValue(column, record.get(nullField));
  assertEquals(NullValue.NULL_VALUE, value.getNullValue());
}
 

/**
 * Tests that BigQueryConverters.columnToValue() returns a Double {@link Value} when the BigQuery
 * column is a FLOAT.
 */
@Test
public void testColumnToValueFloat() {
  TableFieldSchema column = new TableFieldSchema().setName(floatField).setType("FLOAT");
  Record record =
      generateSingleFieldAvroRecord(floatField, "float", floatFieldDesc, floatFieldValue);
  Value value = BigQueryConverters.columnToValue(column, record.get(floatField));
  assertEquals(floatFieldValue, value.getDoubleValue(), 0.001);
}
 

/**
 * Tests that BigQueryConverters.columnToValue() returns an Double {@link Value} when the BigQuery
 * column is a FLOAT64.
 */
@Test
public void testColumnToValueFloat64() {
  TableFieldSchema column = new TableFieldSchema().setName(float64Field).setType("FLOAT64");
  Record record =
      generateSingleFieldAvroRecord(float64Field, "float", float64FieldDesc, float64FieldValue);
  Value value = BigQueryConverters.columnToValue(column, record.get(float64Field));
  assertEquals(float64FieldValue, value.getDoubleValue(), 0.001);
}
 

/** Unit test for {@link DatastoreConverters.CheckSameKey}. */
@Test
@Category(NeedsRunner.class)
public void testCheckSameKey() throws Exception {
  Entity dupKeyEntity = Entity.newBuilder()
      .setKey(entities.get(0).getKey())
      .putProperties("SomeBSProp", Value.newBuilder().setStringValue("Some BS Value").build())
      .build();

  // copy all entities
  ArrayList<Entity> testEntitiesWithConflictKey  = new ArrayList<>(entities);

  // Add the duplicate entity at the end of the list
  testEntitiesWithConflictKey.add(dupKeyEntity);

  List<String> expectedErrors = new ArrayList<>();
  EntityJsonPrinter entityJsonPrinter = new EntityJsonPrinter();
  for (Entity e : Arrays.asList(entities.get(0), dupKeyEntity)) {
    expectedErrors.add(ErrorMessage.newBuilder()
        .setMessage("Duplicate Datastore Key")
        .setData(entityJsonPrinter.print(e))
        .build()
        .toJson());
  }

  TupleTag<Entity> goodTag = new TupleTag<Entity>("entities"){};
  TupleTag<String> errorTag = new TupleTag<String>("errors"){};

  PCollectionTuple results = pipeline
      .apply("Create", Create.of(testEntitiesWithConflictKey))
      .apply("RemoveDupKeys", CheckSameKey.newBuilder()
          .setGoodTag(goodTag)
          .setErrorTag(errorTag)
          .build());

  PAssert.that(results.get(goodTag)).containsInAnyOrder(entities.subList(1, entities.size()));
  PAssert.that(results.get(errorTag)).containsInAnyOrder(expectedErrors);

  pipeline.run();
}