com.google.cloud.datastore.Query Java Examples

/**
 * Removes all deleted blobs tracked in this index.
 *
 */
void removeData() {
  log.warn("removing all entries in the index of soft-deleted blobs...");
  Query<Entity> query = Query.newEntityQueryBuilder()
      .setNamespace(namespace)
      .setKind(DELETED_BLOBS)
      .build();
  QueryResults<Entity> results = gcsDatastore.run(query);
  List<Key> keys = StreamSupport.stream(
      Spliterators.spliteratorUnknownSize(results, Spliterator.ORDERED),
      false).map(entity -> entity.getKey()).collect(Collectors.toList());

  // datastore has a hard limit of 500 keys in a single delete
  List<List<Key>> partitions = Lists.partition(keys, 500);
  partitions.stream().forEach(partition -> gcsDatastore.delete(partition.toArray(new Key[partition.size()])) );

  log.warn("deleted {} blobIds from the soft-deleted blob index", keys.size());
}
 

@Override
public Result<Book> listBooks(String startCursorString) {
  Cursor startCursor = null;
  if (startCursorString != null && !startCursorString.equals("")) {
    startCursor = Cursor.fromUrlSafe(startCursorString);    // Where we left off
  }
  Query<Entity> query = Query.newEntityQueryBuilder()       // Build the Query
      .setKind("Book5")                                     // We only care about Books
      .setLimit(10)                                         // Only show 10 at a time
      .setStartCursor(startCursor)                          // Where we left off
      .setOrderBy(OrderBy.asc(Book.TITLE))                  // Use default Index "title"
      .build();
  QueryResults<Entity> resultList = datastore.run(query);   // Run the query
  List<Book> resultBooks = entitiesToBooks(resultList);     // Retrieve and convert Entities
  Cursor cursor = resultList.getCursorAfter();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toUrlSafe();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Override
public Result<Book> listBooks(String startCursorString) {
  Cursor startCursor = null;
  if (startCursorString != null && !startCursorString.equals("")) {
    startCursor = Cursor.fromUrlSafe(startCursorString);    // Where we left off
  }
  Query<Entity> query = Query.newEntityQueryBuilder()       // Build the Query
      .setKind("Book3")                                     // We only care about Books
      .setLimit(10)                                         // Only show 10 at a time
      .setStartCursor(startCursor)                          // Where we left off
      .setOrderBy(OrderBy.asc(Book.TITLE))                  // Use default Index "title"
      .build();
  QueryResults<Entity> resultList = datastore.run(query);   // Run the query
  List<Book> resultBooks = entitiesToBooks(resultList);     // Retrieve and convert Entities
  Cursor cursor = resultList.getCursorAfter();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toUrlSafe();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Test
public void testGetKindNs_Namespace_KindName() {
  final String namespace = "junit";
  final String kindName = "StringField";
  StatKindNs statEntity = stats.getKindNs(namespace, kindName);
  Query query = Query.newEntityQueryBuilder().setNamespace(namespace)
      .setKind(StatConstants.STAT_KIND_NS)
      .setFilter(StructuredQuery.PropertyFilter.eq(StatConstants.PROP_KIND_NAME, kindName))
      .build();
  QueryResults<com.google.cloud.datastore.Entity> results = datastore.run(query);
  com.google.cloud.datastore.Entity nativeEntity = null;
  if (results.hasNext()) {
    nativeEntity = results.next();
  }
  assertTrue(equals(statEntity, nativeEntity));
}
 

/**
 * Initializes Datastore and cleans out any residual values.  Also initializes global variables
 * used for testing.
 */
@Before
public void setUp() {
  datastore = HELPER.getOptions().toBuilder().setNamespace("ghijklmnop").build().getService();
  StructuredQuery<Key> query = Query.newKeyQueryBuilder().build();
  QueryResults<Key> result = datastore.run(query);
  datastore.delete(Iterators.toArray(result, Key.class));
  keyFactory = datastore.newKeyFactory().setKind("Task");
  taskKey = keyFactory.newKey("some-arbitrary-key");
  testEntity = Entity.newBuilder(taskKey, TEST_FULL_ENTITY).build();
  Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
  calendar.set(1990, JANUARY, 1);
  startDate = Timestamp.of(calendar.getTime());
  calendar.set(2000, JANUARY, 1);
  endDate = Timestamp.of(calendar.getTime());
  calendar.set(1999, DECEMBER, 31);
  includedDate = Timestamp.of(calendar.getTime());
}
 

/**
 * Create a new session usable by Tomcat, from a serialized session in a Datastore Entity.
 * @param sessionKey The key associated with the session metadata and attributes.
 * @return A new session containing the metadata and attributes stored in the entity.
 * @throws ClassNotFoundException Thrown if a class serialized in the entity is not available in
 *                                this context.
 * @throws IOException Thrown when an error occur during the deserialization.
 */
private DatastoreSession deserializeSession(Key sessionKey)
    throws ClassNotFoundException, IOException {
  TraceContext loadingSessionContext = startSpan("Fetching the session from Datastore");
  Iterator<Entity> entities = datastore.run(Query.newEntityQueryBuilder()
      .setKind(sessionKind)
      .setFilter(PropertyFilter.hasAncestor(sessionKey))
      .build());
  endSpan(loadingSessionContext);

  DatastoreSession session = null;
  if (entities.hasNext()) {
    session = (DatastoreSession) manager.createEmptySession();
    TraceContext deserializationContext = startSpan("Deserialization of the session");
    session.restoreFromEntities(sessionKey, Lists.newArrayList(entities));
    endSpan(deserializationContext);
  }
  return session;
}
 

private EntityQuery.Builder backfillQueryBuilder(boolean showAll, Filter... filters) {
  final EntityQuery.Builder queryBuilder = Query.newEntityQueryBuilder().setKind(KIND_BACKFILL);

  final List<Filter> andedFilters = Lists.newArrayList(filters);

  if (!showAll) {
    andedFilters.add(PropertyFilter.eq(PROPERTY_ALL_TRIGGERED, false));
    andedFilters.add(PropertyFilter.eq(PROPERTY_HALTED, false));
  }

  if (!andedFilters.isEmpty()) {
    final Filter head = andedFilters.get(0);
    final Filter[] tail = andedFilters.stream().skip(1).toArray(Filter[]::new);
    queryBuilder.setFilter(CompositeFilter.and(head, tail));
  }

  return queryBuilder;
}
 

/** Example of running a query to find all entities of one kind. */
// [TARGET run(Query, ReadOption...)]
// [VARIABLE "my_kind"]
public List<Entity> runQuery(String kind) {
  // TODO change so that it's not necessary to hold the entities in a list for integration testing
  // [START runQuery]
  StructuredQuery<Entity> query = Query.newEntityQueryBuilder().setKind(kind).build();
  QueryResults<Entity> results = datastore.run(query);
  List<Entity> entities = Lists.newArrayList();
  while (results.hasNext()) {
    Entity result = results.next();
    // do something with result
    entities.add(result);
  }
  // [END runQuery]
  return entities;
}
 

@Override
public Result<Book> listBooksByUser(String userId, String startCursorString) {
  Cursor startCursor = null;
  if (startCursorString != null && !startCursorString.equals("")) {
    startCursor = Cursor.fromUrlSafe(startCursorString);    // Where we left off
  }
  Query<Entity> query = Query.newEntityQueryBuilder()          // Build the Query
      .setKind("Book5")                                        // We only care about Books
      .setFilter(PropertyFilter.eq(Book.CREATED_BY_ID, userId))// Only for this user
      .setLimit(10)                                            // Only show 10 at a time
      .setStartCursor(startCursor)                             // Where we left off
      // a custom datastore index is required since you are filtering by one property
      // but ordering by another
      .setOrderBy(OrderBy.asc(Book.TITLE))
      .build();
  QueryResults<Entity> resultList = datastore.run(query);   // Run the Query
  List<Book> resultBooks = entitiesToBooks(resultList);     // Retrieve and convert Entities
  Cursor cursor = resultList.getCursorAfter();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toUrlSafe();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Override
public void init(FilterConfig config) throws ServletException {
  // initialize local copy of datastore session variables

  datastore = DatastoreOptions.getDefaultInstance().getService();
  keyFactory = datastore.newKeyFactory().setKind("SessionVariable");
  // Delete all sessions unmodified for over two days
  DateTime dt = DateTime.now(DateTimeZone.UTC);
  Query<Entity> query = Query.newEntityQueryBuilder()
      .setKind("SessionVariable")
      .setFilter(PropertyFilter.le("lastModified", dt.minusDays(2).toString(dtf)))
      .build();
  QueryResults<Entity> resultList = datastore.run(query);
  while (resultList.hasNext()) {
    Entity stateEntity = resultList.next();
    datastore.delete(stateEntity.getKey());
  }
}
 

protected void deleteSessionWithValue(String varName, String varValue) {
  Transaction transaction = datastore.newTransaction();
  try {
    Query<Entity> query = Query.newEntityQueryBuilder()
        .setKind("SessionVariable")
        .setFilter(PropertyFilter.eq(varName, varValue))
        .build();
    QueryResults<Entity> resultList = transaction.run(query);
    while (resultList.hasNext()) {
      Entity stateEntity = resultList.next();
      transaction.delete(stateEntity.getKey());
    }
    transaction.commit();
  } finally {
    if (transaction.isActive()) {
      transaction.rollback();
    }
  }
}
 

@Override
public Result<Book> listBooks(String startCursorString) {
  Cursor startCursor = null;
  if (startCursorString != null && !startCursorString.equals("")) {
    startCursor = Cursor.fromUrlSafe(startCursorString);    // Where we left off
  }
  Query<Entity> query = Query.newEntityQueryBuilder()       // Build the Query
      .setKind("Book2")                                     // We only care about Books
      .setLimit(10)                                         // Only show 10 at a time
      .setStartCursor(startCursor)                          // Where we left off
      .setOrderBy(OrderBy.asc(Book.TITLE))                  // Use default Index "title"
      .build();
  QueryResults<Entity> resultList = datastore.run(query);   // Run the query
  List<Book> resultBooks = entitiesToBooks(resultList);     // Retrieve and convert Entities
  Cursor cursor = resultList.getCursorAfter();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toUrlSafe();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Override
public void run(Transaction tx, Key userKey, Void arg) {
  Entity user = tx.get(userKey);
  if (user == null) {
    System.out.println("Nothing to delete, user does not exist.");
    return;
  }
  Query<Key> query =
      Query.newKeyQueryBuilder()
          .setNamespace(NAMESPACE)
          .setKind(COMMENT_KIND)
          .setFilter(PropertyFilter.hasAncestor(userKey))
          .build();
  QueryResults<Key> comments = tx.run(query);
  int count = 0;
  while (comments.hasNext()) {
    tx.delete(comments.next());
    count++;
  }
  tx.delete(userKey);
  System.out.printf("Deleting user '%s' and %d comment[s].%n", userKey.getName(), count);
}
 

@Override
public List<String> retrieveClaims(int limit) {
  // Get claim saved in Datastore
  Datastore datastore = getDatastoreInstance();
  Query<Entity> query = Query.newEntityQueryBuilder().setKind(claimsKind).setLimit(limit).build();
  QueryResults<Entity> results = datastore.run(query);

  List<String> claims = new ArrayList<>();
  while (results.hasNext()) {
    Entity entity = results.next();
    String claim = entity.getString("claim");
    if (claim != null) {
      claims.add(claim);
    }
  }
  return claims;
}
 

protected void deleteSessionWithValue(String varName, String varValue) {
  Transaction transaction = datastore.newTransaction();
  try {
    Query<Entity> query = Query.newEntityQueryBuilder()
        .setKind("SessionVariable")
        .setFilter(PropertyFilter.eq(varName, varValue))
        .build();
    QueryResults<Entity> resultList = transaction.run(query);
    while (resultList.hasNext()) {
      Entity stateEntity = resultList.next();
      transaction.delete(stateEntity.getKey());
    }
    transaction.commit();
  } finally {
    if (transaction.isActive()) {
      transaction.rollback();
    }
  }
}
 

@Override
public void init(FilterConfig config) throws ServletException {
  // initialize local copy of datastore session variables

  datastore = DatastoreOptions.getDefaultInstance().getService();
  keyFactory = datastore.newKeyFactory().setKind("SessionVariable");
  // Delete all sessions unmodified for over two days
  DateTime dt = DateTime.now(DateTimeZone.UTC);
  Query<Entity> query = Query.newEntityQueryBuilder()
      .setKind("SessionVariable")
      .setFilter(PropertyFilter.le("lastModified", dt.minusDays(2).toString(dtf)))
      .build();
  QueryResults<Entity> resultList = datastore.run(query);
  while (resultList.hasNext()) {
    Entity stateEntity = resultList.next();
    datastore.delete(stateEntity.getKey());
  }
}
 

@Override
public List<String> getKinds(boolean excludeSystemKinds) {
  try {
    String query = "SELECT __key__ FROM " + ENTITY_KINDS + " ORDER BY __key__";
    GqlQuery.Builder<Key> gqlQueryBuilder = Query.newGqlQueryBuilder(ResultType.KEY, query);
    gqlQueryBuilder.setNamespace(entityManager.getEffectiveNamespace());
    GqlQuery<Key> gqlQuery = gqlQueryBuilder.build();
    QueryResults<Key> results = entityManager.getDatastore().run(gqlQuery);
    List<String> kinds = new ArrayList<>(50);
    while (results.hasNext()) {
      Key key = results.next();
      String kind = key.getName();
      if (excludeSystemKinds && kind.startsWith("__")) {
        continue;
      }
      kinds.add(key.getName());
    }
    return kinds;
  } catch (DatastoreException exp) {
    throw new EntityManagerException(exp);
  }
}
 

@Override
public Result<Book> listBooks(String startCursorString) {
  Cursor startCursor = null;
  if (startCursorString != null && !startCursorString.equals("")) {
    startCursor = Cursor.fromUrlSafe(startCursorString);    // Where we left off
  }
  Query<Entity> query = Query.newEntityQueryBuilder()       // Build the Query
      .setKind("Book4")                                     // We only care about Books
      .setLimit(10)                                         // Only show 10 at a time
      .setStartCursor(startCursor)                          // Where we left off
      .setOrderBy(OrderBy.asc(Book.TITLE))                  // Use default Index "title"
      .build();
  QueryResults<Entity> resultList = datastore.run(query);   // Run the query
  List<Book> resultBooks = entitiesToBooks(resultList);     // Retrieve and convert Entities
  Cursor cursor = resultList.getCursorAfter();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toUrlSafe();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Override
public List<String> retrieveTokens(int limit) {
  // Get token saved in Datastore
  Datastore datastore = getDatastoreInstance();
  Query<Entity> query = Query.newEntityQueryBuilder().setKind(tokensKind).setLimit(limit).build();
  QueryResults<Entity> results = datastore.run(query);

  List<String> tokens = new ArrayList<>();
  while (results.hasNext()) {
    Entity entity = results.next();
    String token = entity.getString("token");
    if (token != null) {
      tokens.add(token);
    }
  }
  return tokens;
}
 

@Override
public void init(FilterConfig config) throws ServletException {
  // initialize local copy of datastore session variables

  datastore = DatastoreOptions.getDefaultInstance().getService();
  keyFactory = datastore.newKeyFactory().setKind("SessionVariable");
  // Delete all sessions unmodified for over two days
  DateTime dt = DateTime.now(DateTimeZone.UTC);
  Query<Entity> query = Query.newEntityQueryBuilder()
      .setKind("SessionVariable")
      .setFilter(PropertyFilter.le("lastModified", dt.minusDays(2).toString(dtf)))
      .build();
  QueryResults<Entity> resultList = datastore.run(query);
  while (resultList.hasNext()) {
    Entity stateEntity = resultList.next();
    datastore.delete(stateEntity.getKey());
  }
}
 

/**
 * In case the return type is a closed projection (only the projection fields are necessary
 * to construct an entity object), this method returns a {@link ProjectionEntityQuery.Builder}
 * with projection fields only.
 * Otherwise returns a {@link EntityQuery.Builder}.
 */
private Builder<?> getEntityOrProjectionQueryBuilder() {
	ProjectionInformation projectionInformation =
			this.projectionFactory.getProjectionInformation(this.queryMethod.getReturnedObjectType());

	if (projectionInformation != null &&
			projectionInformation.getType() != this.entityType
			&& projectionInformation.isClosed()) {
		ProjectionEntityQuery.Builder projectionEntityQueryBuilder = Query.newProjectionEntityQueryBuilder();
		projectionInformation.getInputProperties().forEach(
				propertyDescriptor ->
						projectionEntityQueryBuilder.addProjection(mapToFieldName(propertyDescriptor)));
		return projectionEntityQueryBuilder;
	}
	return StructuredQuery.newEntityQueryBuilder();
}
 

@Test
public void testNamespaceRunQuery() {
  setUpQueryTests();
  // [START datastore_namespace_run_query]
  KeyFactory keyFactory = datastore.newKeyFactory().setKind("__namespace__");
  Key startNamespace = keyFactory.newKey("g");
  Key endNamespace = keyFactory.newKey("h");
  Query<Key> query = Query.newKeyQueryBuilder()
      .setKind("__namespace__")
      .setFilter(CompositeFilter.and(
          PropertyFilter.gt("__key__", startNamespace),
          PropertyFilter.lt("__key__", endNamespace)))
      .build();
  List<String> namespaces = new ArrayList<>();
  QueryResults<Key> results = datastore.run(query);
  while (results.hasNext()) {
    namespaces.add(results.next().getName());
  }
  // [END datastore_namespace_run_query]
  assertEquals(ImmutableList.of("ghijklmnop"), namespaces);
}
 

@Test
public void testPropertyRunQuery() {
  setUpQueryTests();
  // [START datastore_property_run_query]
  Query<Key> query = Query.newKeyQueryBuilder().setKind("__property__").build();
  QueryResults<Key> keys = datastore.run(query);
  Map<String, Collection<String>> propertiesByKind = new HashMap<>();
  while (keys.hasNext()) {
    Key key = keys.next();
    String kind = key.getParent().getName();
    String propertyName = key.getName();
    Collection<String> properties = propertiesByKind.get(kind);
    if (properties == null) {
      properties = new HashSet<>();
      propertiesByKind.put(kind, properties);
    }
    properties.add(propertyName);
  }
  // [END datastore_property_run_query]
  Map<String, ImmutableSet<String>> expected = ImmutableMap.of("Task", ImmutableSet.of(
      "done", "category", "done", "completed", "priority", "created", "percent_complete", "tag"));
  assertEquals(expected, propertiesByKind);
}
 

public static void wipeDatastore() {
  Datastore datastore = getDatastore();
  QueryResults<Key> guestbooks =
      datastore.run(Query.newKeyQueryBuilder().setKind("Greeting").build());
  ArrayList<Key> keys = Lists.newArrayList(guestbooks);

  if (!keys.isEmpty()) {
    datastore.delete(keys.toArray(new Key[keys.size()]));
  }
}
 

@Test
public void testGetKind() {
  final String kindName = "StringField";
  StatKind statEntity = stats.getKind(kindName);
  Query query = Query.newEntityQueryBuilder().setNamespace("").setKind(StatConstants.STAT_KIND)
      .setFilter(StructuredQuery.PropertyFilter.eq(StatConstants.PROP_KIND_NAME, kindName))
      .build();
  QueryResults<com.google.cloud.datastore.Entity> results = datastore.run(query);
  com.google.cloud.datastore.Entity nativeEntity = null;
  if (results.hasNext()) {
    nativeEntity = results.next();
  }
  assertTrue(equals(statEntity, nativeEntity));

}
 

/** Example of creating and running a typed GQL query. */
// [TARGET newGqlQueryBuilder(ResultType, String)]
// [VARIABLE "my_kind"]
public QueryResults<Entity> newTypedQuery(String kind) {
  // [START newTypedQuery]
  String gqlQuery = "select * from " + kind;
  Query<Entity> query = Query.newGqlQueryBuilder(Query.ResultType.ENTITY, gqlQuery).build();
  QueryResults<Entity> results = datastore.run(query);
  // Use results
  // [END newTypedQuery]
  return results;
}
 

@Test
public void testPropertyFilter() {
  setUpQueryTests();
  // [START datastore_property_filter]
  Query<Entity> query =
      Query.newEntityQueryBuilder().setKind("Task").setFilter(PropertyFilter.eq("done", false))
      .build();
  // [END datastore_property_filter]
  assertValidQuery(query);
}
 

@Test
public void testGetCompositeIndexesNs() {
  List<StatCompositeIndexNs> statEntities = stats.getCompositeIndexesNs();
  Query query = Query.newEntityQueryBuilder().setKind(StatConstants.STAT_COMPOSITE_INDEX_NS)
      .build();
  QueryResults<com.google.cloud.datastore.Entity> results = datastore.run(query);
  int i = 0;
  while (results.hasNext()) {
    StatCompositeIndexNs statEntity = statEntities.get(i);
    com.google.cloud.datastore.Entity nativeEntity = results.next();
    assertTrue(equals(statEntity, nativeEntity));
    i++;
  }
}
 

@Test
public void testRunQuery() {
  setUpQueryTests();
  Query<Entity> query = Query.newEntityQueryBuilder().setKind("Task").build();
  // [START datastore_run_query]
  QueryResults<Entity> tasks = datastore.run(query);
  // [END datastore_run_query]
  assertNotNull(tasks.next());
  assertFalse(tasks.hasNext());
}
 

@Test
public void testCompositeFilter() {
  setUpQueryTests();
  // [START datastore_composite_filter]
  Query<Entity> query = Query.newEntityQueryBuilder()
      .setKind("Task")
      .setFilter(
          CompositeFilter.and(PropertyFilter.eq("done", false), PropertyFilter.eq("priority", 4)))
      .build();
  // [END datastore_composite_filter]
  assertValidQuery(query);
}