org.apache.jena.riot.lang.PipedRDFIterator Java Examples

public static List<String[]> getSubjectAndObjectsFromNTriple(String filename, String replacePrefix) {

		List<String[]> results = new ArrayList<String[]>();

		PipedRDFIterator<Triple> iter = fileToStreamIterator(filename);

		while (iter.hasNext()) {
			Triple statement = iter.next();
			results.add(new String[] {
					replacePrefix == null || "".equals(replacePrefix) ? statement.getSubject().getURI()
							: statement.getSubject().getURI().replace(replacePrefix, ""),
					replacePrefix == null || "".equals(replacePrefix) ? statement.getObject().getURI()
							: statement.getObject().getURI().replace(replacePrefix, ""), });

		}

		iter.close();
		return null;

		// return results;
	}
 

public static List<String> getSubjectsFromNTriple(String filename, String replacePrefix) {

		List<String> results = new ArrayList<String>();

		PipedRDFIterator<Triple> iter = fileToStreamIterator(filename);
		int i = 0;
		while (iter.hasNext()) {
			Triple statement = iter.next();
			i++;
			if (i % 10000 == 0) {
				log.debug("still reading " + filename);
			}
			results.add(replacePrefix == null || "".equals(replacePrefix) ? statement.getSubject().getURI()
					: statement.getSubject().getURI().replace(replacePrefix, ""));

			
		}
		iter.close();

		return results;
	}
 

private static PipedRDFIterator<Triple> fileToStreamIterator(String filename) {
	PipedRDFIterator<Triple> iter = new PipedRDFIterator<>();
	final PipedRDFStream<Triple> inputStream = new PipedTriplesStream(iter);

	// PipedRDFStream and PipedRDFIterator need to be on different threads
	ExecutorService executor = Executors.newSingleThreadExecutor();

	// Create a runnable for our parser thread
	Runnable parser = new Runnable() {

		@Override
		public void run() {
			RDFDataMgr.parse(inputStream, filename);
		}
	};

	// Start the parser on another thread
	executor.submit(parser);
	// We will consume the input on the main thread here
	// We can now iterate over data as it is parsed, parsing only runs as
	// far ahead of our consumption as the buffer size allows
	return iter;
}
 

public void execute()
{
    // print all prefixes and the base IRI to the output file
    try ( IndentedWriter writer = new IndentedWriter(outStream) ) {
        RiotLib.writePrefixes(writer, pmap);
        RiotLib.writeBase(writer, baseIRI);

        // second pass over the file to perform the conversion in a streaming manner
        final PipedRDFIterator<Triple> it = new PipedRDFIterator<>(BUFFER_SIZE);
        final PipedTriplesStream triplesStream = new PipedTriplesStream(it);

        // PipedRDFStream and PipedRDFIterator need to be on different threads
        final Runnable r = new Runnable() {
            @Override
            public void run() {
                RDFParser.create().labelToNode( LabelToNode.createUseLabelEncoded() )
                .source(inputFilename)
                .lang(LangTurtleStar.TURTLESTAR)
                .base(baseIRI)
                .build()
                .parse(triplesStream);
            }
        };
        final ExecutorService executor = Executors.newSingleThreadExecutor();
        executor.submit(r);

        final NodeFormatter nFmt = new NodeFormatterTTL(baseIRI, pmap);

        while ( it.hasNext() ) {
            printTriples(it.next(), writer, nFmt, false);
        }

        it.close();
        executor.shutdown();

        writer.write(" .");
        writer.flush();
    }
}
 

public void execute()
{
	final PipedRDFIterator<Triple> it = new PipedRDFIterator<>(BUFFER_SIZE);
	final PipedTriplesStream triplesStream = new PipedTriplesStream(it);

	// PipedRDFStream and PipedRDFIterator need to be on different threads
	final Runnable r = new Runnable() {
		@Override
		public void run() {
			RDFParser.create().labelToNode(LabelToNode.createUseLabelEncoded())
                  .source(inputFilename)
                  .lang(LangTurtleStar.TURTLESTAR)
                  .base(baseIRI)
                  .build()
                  .parse(triplesStream);
		}
	};
	final ExecutorService executor = Executors.newSingleThreadExecutor();
	executor.submit(r);

	// consume the iterator
	while ( it.hasNext() ) {
       	it.next();
       }

	pmap = it.getPrefixes();
	if(baseIRI == null) {
		baseIRI = it.getBaseIri();
	}

	it.close();
	executor.shutdown();
}
 

@Override
public void load( 
		final SolrQueryRequest request, 
		final SolrQueryResponse response,
		final ContentStream stream, 
		final UpdateRequestProcessor processor) throws Exception {
	
	final PipedRDFIterator<Quad> iterator = new PipedRDFIterator<Quad>();
	final StreamRDF inputStream = new PipedQuadsStream(iterator);
	
	executor.submit(new Runnable() {
		@Override
		public void run() {
			try {
				RDFDataMgr.parse(
						inputStream, 
						stream.getStream(), 
						RDFLanguages.contentTypeToLang(stream.getContentType()));
			} catch (final IOException exception) {
				throw new SolrException(ErrorCode.SERVER_ERROR, exception);
			}					
		}
	});
	
	final DatasetGraph dataset = new LocalDatasetGraph(request, response);
	while (iterator.hasNext()) {
		dataset.add(iterator.next());
	}									
}
 

@Override
public void load(
		final SolrQueryRequest request, 
		final SolrQueryResponse response,
		final ContentStream stream, 
		final UpdateRequestProcessor processor) throws Exception {
	
	final PipedRDFIterator<Triple> iterator = new PipedRDFIterator<Triple>();
	final StreamRDF inputStream = new PipedTriplesStream(iterator);
	
	executor.submit(new Runnable() {
		@Override
		public void run() {
			try {
				RDFDataMgr.parse(
						inputStream, 
						stream.getStream(), 
						RDFLanguages.contentTypeToLang(stream.getContentType()));
			} catch (final IOException exception) {
				throw new SolrException(ErrorCode.SERVER_ERROR, exception);
			}					
		}
	});
		
	// Graph Store Protocol indicates the target graph URI separately.
	// So the incoming Content-type here is one that maps "Triples Loader" but
	// the indexed tuple could be a Quad.
	final String graphUri = request.getParams().get(Names.GRAPH_URI_ATTRIBUTE_NAME);
	
	final DatasetGraph dataset = new LocalDatasetGraph(request, response, null, null);
	final Graph defaultGraph = graphUri == null 
			? dataset.getDefaultGraph() 
			: dataset.getGraph(NodeFactory.createURI(graphUri));
	while (iterator.hasNext()) {
		defaultGraph.add(iterator.next());
	}		
}
 

public static void main(String... argv) {
    final String filename = "data.ttl";

    // Create a PipedRDFStream to accept input and a PipedRDFIterator to
    // consume it
    // You can optionally supply a buffer size here for the
    // PipedRDFIterator, see the documentation for details about recommended
    // buffer sizes
    PipedRDFIterator<Triple> iter = new PipedRDFIterator<Triple>();
    final PipedRDFStream<Triple> inputStream = new PipedTriplesStream(iter);

    // PipedRDFStream and PipedRDFIterator need to be on different threads
    ExecutorService executor = Executors.newSingleThreadExecutor();

    // Create a runnable for our parser thread
    Runnable parser = new Runnable() {

        @Override
        public void run() {
            // Call the parsing process.
            RDFDataMgr.parse(inputStream, filename);
        }
    };

    // Start the parser on another thread
    executor.submit(parser);

    // We will consume the input on the main thread here

    // We can now iterate over data as it is parsed, parsing only runs as
    // far ahead of our consumption as the buffer size allows
    while (iter.hasNext()) {
        Triple next = iter.next();
        // Do something with each triple
    }
}
 

public void convert( String inputFilename,
		             OutputStream outStream,
                        Lang inputLang,
		             String baseIRI,
		             ErrorHandler errHandler,
		             boolean enableChecking )
{
	final FirstPass fp = new FirstPass(inputFilename, inputLang, baseIRI, errHandler, enableChecking);
	fp.execute();

	// print all prefixes and the base IRI to the output file
	try( IndentedWriter writer = new IndentedWriter(outStream) ) {
	    RiotLib.writePrefixes(writer, fp.getPrefixMap());
	    RiotLib.writeBase(writer, fp.getBaseIRI());

	    // second pass over the file to perform the conversion in a streaming manner
	    // (PipedTriplesStream and PipedRDFIterator need to be on different threads!!)
	    final PipedRDFIterator<Triple> it = new PipedRDFIterator<>(BUFFER_SIZE);
	    final PipedTriplesStream triplesStream = new PipedTriplesStream(it);

	    final Parser p = new Parser(inputFilename, triplesStream, enableChecking);

	    if ( baseIRI != null )
	        p.setBaseIRI(baseIRI);

	    if ( inputLang != null )
	        p.setLang(inputLang);

	    if ( errHandler != null )
	        p.setErrorHandler(errHandler);

	    final ExecutorService executor = Executors.newSingleThreadExecutor();
	    executor.submit(p);

	    final NodeFormatter nFmt = new NodeFormatterTurtleStarExtImpl(fp.getBaseIRI(), fp.getPrefixMap());
	    printTriples(writer, nFmt, it, fp.getReifiedTriples());

	    it.close();
	    executor.shutdown();

	    writer.write(" .");
	    writer.flush();
	}
}