org.apache.jena.sparql.core.Quad Java Examples

/**
 * filter RDD of {@link Quad}s based on the specified config
 *
 * @param quads RDD of quads to filter
 * @return filtered RDD of quads
 */
public JavaRDD<Quad> filter(JavaRDD<Quad> quads) {
    Set<String> subjectBlacklist = new HashSet<>(config.getResourceBlacklist());
    if (config.getResources().isEmpty()) {
        return QuadUtils.filterQuadsByForbiddenSubjects(quads, subjectBlacklist);
    }
    log.info("Filtering quads");
    Set<String> subjects = new HashSet<>(config.getResources());
    boolean directed = config.isDirected();
    for (int d = 0; d < config.getRelatedDepth(); d++) {
        log.info("Depth {}, collecting neighbors of {} resources", d, subjects.size());
        List<String> neighbors = QuadUtils.getNeighborResources(quads, subjects, directed).collect();
        subjects.addAll(neighbors);
        subjects.removeAll(subjectBlacklist);
    }
    log.info("Filtering on an in-memory set of {} subjects", subjects.size());
    quads = QuadUtils.filterQuadsByAllowedSubjects(quads, subjects);
    return quads;
}
 

static void deleteAny(DatasetGraph dsg, Node g, Node s, Node p, Node o, int sliceSize) {
    // Delete in slices rather than assume .remove() on the iterator is implemented.
    // We keep executing find(g, s, p, o) until we don't get a full slice.
    Quad[] buffer = new Quad[DeleteBufferSize];
    while (true) {
        Iterator<Quad> iter = dsg.find(g, s, p, o);
        // Get a slice
        int len = 0;
        for ( ; len < DeleteBufferSize ; len++ ) {
            if ( !iter.hasNext() )
                break;
            buffer[len] = iter.next();
        }
        // Delete them.
        for ( int i = 0 ; i < len ; i++ ) {
            dsg.delete(buffer[i]);
            buffer[i] = null;
        }
        // Finished?
        if ( len < DeleteBufferSize )
            break;
    }
}
 

@Test
public void change_2() {
    String NAME = "change_2";
    DeltaClient dClient = createRegister(NAME);
    try(DeltaConnection dConn = dClient.get(NAME)) {
        Id dsRef = dConn.getDataSourceId();
        Version version = dConn.getRemoteVersionLatest();

        DatasetGraph dsg = dConn.getDatasetGraph();
        Txn.executeWrite(dsg, ()->{
            Quad q = SSE.parseQuad("(_ :s1 :p1 :o1)");
            dsg.add(q);
        });
        // Rebuild directly.
        DatasetGraph dsg2 = DatasetGraphFactory.createTxnMem();
        Version ver = dConn.getRemoteVersionLatest();
        RDFPatch patch1 = dConn.getLink().fetch(dsRef, ver) ;
        RDFPatchOps.applyChange(dsg2, patch1);

        Set<Quad> set1 = Txn.calculateRead(dsg, ()->Iter.toSet(dsg.find()));
        Set<Quad> set2 = Txn.calculateRead(dsg2, ()->Iter.toSet(dsg2.find()));

        assertEquals(set1, set2);
    }
}
 

/**
 * Get map of rdfs:labels for specified URIs
 *
 * @param quads Quads to use for retrieving labels
 * @param uris  Set of URI Strings to find labels for
 * @return map of URI -> rdfs:label
 */
private Map<String, String> getRdfsLabels(JavaRDD<Quad> quads, Set<String> uris) {
    Broadcast<Set<String>> broadcastURIs = sc.broadcast(uris);
    Map<String, String> nonSerializableMap = quads.filter(quad ->
                    // filter out label predicates for specified subject URIs
                    quad.getPredicate().isURI() &&
                            quad.getPredicate().getURI().equals(LABEL_URI) &&
                            quad.getSubject().isURI() &&
                            (broadcastURIs.getValue().contains(quad.getSubject().getURI()))
            // map to pair of uri, label
    ).mapToPair(quad -> new Tuple2<>(
            quad.getSubject().getURI(),
            quad.getObject().getLiteralValue().toString()
    )).collectAsMap();

    return new HashMap<>(nonSerializableMap);
}
 

@Test public void changeSuppressEmptyCommit_4() {
    Quad q = SSE.parseQuad("(_ :s :p 'object')");
    Triple t = SSE.parseTriple("(:t :p 'object')");

    Txn.executeRead(dsg,   ()->{});
    testCounters(counter.summary(), 0, 0);

    Txn.executeWrite(dsg,  ()->{dsg.add(q);});
    testCounters(counter.summary(), 1, 0);

    Txn.executeWrite(dsg,  ()->{dsg.getDefaultGraph().add(t);});
    testCounters(counter.summary(), 2, 0);

    Txn.executeWrite(dsg,  ()->{dsg.getDefaultGraph().getPrefixMapping().setNsPrefix("", "http://example/");});
    testCounters(counter.summary(), 2, 1);

    Txn.executeWrite(dsg,  ()->{});
    testCounters(counter.summary(), 2, 1);
}
 

private void change_read_new(Runnable betweenSections) {
    Quad quad = DeltaTestLib.freshQuad();

    String NAME = "DS-"+counter.incrementAndGet();
    DeltaClient dClient = createRegister(NAME);
    Id dsRef;

    try(DeltaConnection dConn = dClient.get(NAME)) {
        dConn.getPatchLogInfo().getDataSourceId();
        dsRef = dConn.getDataSourceId();
        Version version = dConn.getRemoteVersionLatest();
        DatasetGraph dsg = dConn.getDatasetGraph();
        Txn.executeWrite(dsg, ()->dsg.add(quad) );
    }

    betweenSections.run();

    // New client.
    // Rebuild.
    dClient = resetDeltaClient(NAME);
    try(DeltaConnection dConn = dClient.get(NAME)) {
        boolean b = dConn.getDatasetGraph().contains(quad);
        assertTrue(b);
    }
}
 

@Test
public void update_2() {
    // Create on the Delta link then setup DeltaClient
    DeltaLink dLink = getLink();
    String DS_NAME = "1234";

    Id dsRef = dLink.newDataSource(DS_NAME, "http://example/datasource_update_2");
    DeltaClient dClient = createDeltaClient();
    dClient.register(dsRef, LocalStorageType.MEM, SyncPolicy.NONE);
    DeltaConnection dConn = dClient.get(DS_NAME);
    assertNotNull(dConn);
    assertEquals(Version.INIT, dConn.getLocalVersion());
    assertEquals(Version.INIT, dConn.getRemoteVersionLatest());

    Quad quad = SSE.parseQuad("(_ :s :p :o)");
    DatasetGraph dsg = dConn.getDatasetGraph();

    long x0 = Txn.calculateRead(dsg, ()->Iter.count(dsg.find()) );
    assertEquals(0, x0);

    Txn.executeWrite(dsg, ()->dsg.add(quad));
    long x1 = Txn.calculateRead(dsg, ()->Iter.count(dsg.find()) );
    assertEquals(1, x1);

    long x2 = Iter.count(dConn.getStorage().find());
    assertEquals(1, x1);
}
 

protected Graph getGraph(Quad quad) {
	if(quad.isDefaultGraph()) {
		return getDefaultGraph();
	}
	else {
		return getGraph(quad.getGraph());
	}
}
 

private Formula handleQuad(Quad arg0) {
	return constrainDomain(toTerm(arg0.getGraph()), Leaf.GRAPH)
	.product(constrainDomain(toTerm(arg0.getSubject()), Leaf.SUBJECT))
	.product(constrainDomain(toTerm(arg0.getPredicate()), Leaf.PREDICATE))
	.product(constrainDomain(toTerm(arg0.getObject()), Leaf.OBJECT))
	.in(QuadTableRelations.quads);
}
 

@Override
public Iterator<Quad> findNG(Node g, Node s, Node p, Node o)
   {
   if ((g == null) || g.equals(Node.ANY))
      {
      // TODO: REVISIT: very inefficient way to get all the matching quads from named graphs only
      LinkedList<Quad> l = new LinkedList<Quad>();
      Iterator<Quad> res = find(g, s, p, o);
      while (res.hasNext())
         {
         Quad q = res.next();
         if (q.getGraph() != null
               && (!q.getGraph().isURI() || !q.getGraph().getURI().equalsIgnoreCase(Constants.DEFAULT_GRAPH_MONIKER)))
            {
            l.add(q);
            }
         }
      return l.iterator();

      }
   else if (g.getURI().equalsIgnoreCase(Constants.DEFAULT_GRAPH_MONIKER))
      {
      return new LinkedList<Quad>().iterator();
      }
   else
      {
      return find(g, s, p, o);
      }
   }
 

/**
 * Get resources related to specified resources, computed by querying an in-memory set of subjects
 *
 * @param quads       RDD of quads to filter
 * @param subjectURIs set of requested subject URIs to grow from
 * @param directed    whether to use both directions of relations
 * @return URIs of resources related to specified resources
 */
public static JavaRDD<String> getNeighborResources(JavaRDD<Quad> quads, Set<String> subjectURIs, boolean directed) {
    JavaRDD<String> neighbors = filterQuadsByAllowedSubjects(quads, subjectURIs)
            .filter(quad -> quad.getObject().isURI())
            .map(quad -> quad.getObject().getURI());
    if (!directed) {
        neighbors = neighbors.union(filterQuadsByObjects(quads, subjectURIs)
                .filter(quad -> quad.getSubject().isURI())
                .map(quad -> quad.getSubject().getURI()));
    }
    return neighbors;
}
 

/**
 * Get quads with specified subjects filtered out, computed by querying an in-memory set of subjects
 *
 * @param quads            RDD of quads to filter
 * @param subjectBlacklist set of requested subject URIs to be filtered out
 * @return filtered RDD with only those quads whose subject is NOT in subjectBlacklist
 */
public static JavaRDD<Quad> filterQuadsByForbiddenSubjects(JavaRDD<Quad> quads, Set<String> subjectBlacklist) {
    if (subjectBlacklist.isEmpty()) {
        return quads;
    }
    return quads.filter(quad -> !quad.getSubject().isURI() ||
            !subjectBlacklist.contains(quad.getSubject().getURI())
    );
}
 

@Override
public void format(Template template) {
    out.print("{");
    out.incIndent(INDENT);
    out.pad();

    List<Quad> quads = template.getQuads();
    for (Quad quad : quads) {
        BasicPattern bgp = new BasicPattern();
        bgp.add(quad.asTriple());
        out.newline();
        if (!Quad.defaultGraphNodeGenerated.equals(quad.getGraph())) {

            out.print("GRAPH");
            out.print(" ");
            out.print(slotToString(quad.getGraph()));
            out.print(" ");

            out.newline();
            out.incIndent(INDENT);
            out.pad();
            out.print("{");
            out.incIndent(INDENT);
            out.pad();
        }

        formatTriples(bgp);

        if (!Quad.defaultGraphNodeGenerated.equals(quad.getGraph())) {
            out.decIndent(INDENT);
            out.print("}");
            out.decIndent(INDENT);
        }
    }
    out.newline();
    out.decIndent(INDENT);
    out.print("}");
    out.newline();
}
 

@Test public void record_add_graph_1() {
    Txn.executeWrite(dsg, ()->dsg.add(quad1));

    Graph data = GraphFactory.createDefaultGraph();
    Node gn = quad1.getGraph();
    data.add(triple1);
    Txn.executeWrite(dsg, ()-> {
        dsg.addGraph(gn, data);
    });
    DatasetGraph dsg2 = replay();
    check(dsg2, Quad.create(gn, triple1));
}
 

@Test public void record_graph_2() {
    Txn.executeWrite(dsg, ()-> {
        Graph g = dsg.getDefaultGraph();
        g.add(triple1);
        g.delete(triple1);
        g.add(triple2);
    });
    DatasetGraph dsg2 = replay();
    check(dsg2, Quad.create(Quad.defaultGraphIRI, triple2));
}
 

@Test public void record_graph_1() {
    Txn.executeWrite(dsg, ()-> {
        Graph g = dsg.getDefaultGraph();
        g.add(triple1);
    });
    DatasetGraph dsg2 = replay();
    check(dsg2, Quad.create(Quad.defaultGraphIRI, triple1));
}
 

@Test
public void update_1() {
    // Create on the Delta link then setup DeltaClient
    DeltaLink dLink = getLink();
    String DS_NAME = "123";

    Id dsRef = dLink.newDataSource(DS_NAME, "http://example/datasource_update_1");
    DeltaClient dClient = createDeltaClient();
    dClient.register(dsRef, LocalStorageType.MEM, SyncPolicy.NONE);
    DeltaConnection dConn = dClient.get(DS_NAME);
    assertNotNull(dConn);
    assertEquals(Version.INIT, dConn.getLocalVersion());
    assertEquals(Version.INIT, dConn.getRemoteVersionLatest());

    Quad quad = SSE.parseQuad("(_ :s :p :o)");
    DatasetGraph dsg = dConn.getDatasetGraph();
    long x0 = Iter.count(dsg.find());
    assertEquals(0, x0);
    Txn.executeWrite(dsg, ()->dsg.add(quad));

    long x1 = Txn.calculateRead(dsg, ()->Iter.count(dsg.find()));
    assertEquals(1, x1);

    DatasetGraph dsgx = dConn.getStorage();
    long x2 = Txn.calculateRead(dsgx, ()->Iter.count(dsgx.find()));
    assertEquals(1, x1);

}
 

/**
 * Parse RDF file from resources folder
 * @param sc spark context to use for parsing
 * @param fileName name of the file to parse
 * @return RDD of quads from the requested file
 */
public static JavaRDD<Quad> getQuadsRDD(JavaSparkContext sc, String fileName) {
    QuadParser parser = new ElephasQuadParser(
            new QuadParserConfig()
                    .setBatchSize(2),
            sc
    );
    String path = TestUtils.getDatasetPath(fileName);
    return parser.parseQuads(path);
}
 

private static void check(DatasetGraph dsg, Quad...quads) {
    if ( quads.length == 0 ) {
        assertTrue(dsg.isEmpty());
        return;
    }

    List<Quad> listExpected = Arrays.asList(quads);
    List<Quad> listActual = Iter.toList(dsg.find());
    assertEquals(listActual.size(), listExpected.size());
    assertTrue(ListUtils.equalsUnordered(listExpected, listActual));
}
 

public static boolean match(Quad quad, Node g, Node s, Node p, Node o) {
    return
        match(quad.getGraph(), g) &&
        match(quad.getSubject(), s) &&
        match(quad.getPredicate(), p) &&
        match(quad.getObject(), o);
}
 

@Override
/** Simple implementation but done without assuming iterator.remove() */
public void deleteAny(Node g, Node s, Node p, Node o) {
    // TODO DRY This code.
    // This is duplicated: see DatasetGraphBase.
    // We need to do the conversion here.
    // DRY => DSGUtils
    // Convert deleteAny to deletes.
    Quad[] buffer = new Quad[DeleteBufferSize];
    while (true) {
        Iterator<Quad> iter = find(g, s, p, o);
        // Get a slice
        int len = 0;
        for ( ; len < DeleteBufferSize ; len++ ) {
            if ( !iter.hasNext() )
                break;
            buffer[len] = iter.next();
        }
        // Delete them.
        for ( int i = 0 ; i < len ; i++ ) {
            delete(buffer[i]);
            buffer[i] = null;
        }
        // Finished?
        if ( len < DeleteBufferSize )
            break;
    }
}
 

private void output(String code, Node g, Node s, Node p, Node o) {
    tok.startTuple();
    outputWord(code);
    output(s);
    output(p);
    output(o);
    if ( g != null && ! Quad.isDefaultGraph(g) )
        output(g);
    tok.endTuple();
    tok.flush();
}
 

public Iterator<Quad> find(Node g, final Node s, final Node p, final Node o)
{

ExtendedIterator<Triple> it;

if ((g == null) || g.equals(Node.ANY))
   {
   it = DB2Graph.find(store, new Triple(s, p, o), null, connection, false, /* not reification */
         true /* search all graphs */);
   }
else
   {

   if (g.getURI().equalsIgnoreCase(Constants.DEFAULT_GRAPH_MONIKER))
      {
      it = defaultModel.getGraph().find(s, p, o);
      }
   else
      {
      it = getGraph(g).find(s, p, o);
      }

   }

Set<Quad> sq = new HashSet<Quad>();
while (it.hasNext())
   {
   sq.add(new Quad(g, it.next()));
   }
it.close();

return sq.iterator();

}
 

public void delete(Quad quad)
{

if (quad.getGraph().getURI().equalsIgnoreCase(Constants.DEFAULT_GRAPH_MONIKER))
   {
   defaultModel.getGraph().delete(quad.asTriple());
   }

getGraph(quad.getGraph()).delete(quad.asTriple());
}
 

@Override
protected void addToDftGraph(Node s, Node p, Node o) {
    if ( get().contains(Quad.defaultGraphIRI, s, p ,o) )
        return;
    Triple triple = Triple.create(s, p, o);
    if ( addTriples.add(triple) )
        deleteTriples.remove(triple);
}
 

@Override
protected void addToNamedGraph(Node g, Node s, Node p, Node o) {
    if ( get().contains(Quad.defaultGraphIRI, s, p ,o) )
        return;
    Triple triple = Triple.create(s, p, o);
    if ( addTriples.add(triple) )
        deleteTriples.remove(triple);
}
 

public boolean contains(Quad quad)
{
Iterator<Quad> it = find(quad);
if (it != null && it.hasNext())
   {
   return true;
   }
return false;
}
 

@Override
public Iterator<Quad> find(Node g, Node s, Node p, Node o) {
    // Original, without deletes
    Iterator<Quad> iter1 = Iter.filter(get().find(g, s, p, o), (q)->!deleteQuads.contains(q) );
    // plus adds (which do not contain deletes)
    Iterator<Quad> iter2 = addQuads.stream().filter(q->Match.match(q, g, s, p, o)).iterator();
    return Iter.concat(iter1,iter2);
}
 

@Override
protected void actionDelete(Node g, Node s, Node p, Node o) {
    if ( checking && ! get().contains(g,s,p,o) )
        return;
    Quad quad = Quad.create(g, s, p, o);
    if ( deleteQuads.add(quad) )
        addQuads.remove(quad);
}
 

@Override
protected void actionAdd(Node g, Node s, Node p, Node o) {
    // add/delete (Quad) creates churn.
    if ( checking && get().contains(g,s,p,o) )
        return;
    Quad quad = Quad.create(g, s, p, o);
    if ( addQuads.add(quad) )
        deleteQuads.remove(quad);
}