Java Code Examples for org.eclipse.rdf4j.rio.Rio#createParser()

private Model parseConfig(File file) throws SailConfigException, IOException
{
    RDFFormat format = Rio.getParserFormatForFileName(file.getAbsolutePath()).get();
    if (format==null)
        throw new SailConfigException("Unsupported file format: " + file.getAbsolutePath());
    RDFParser parser = Rio.createParser(format);
    Model model = new LinkedHashModel();
    parser.setRDFHandler(new StatementCollector(model));
    InputStream stream = new FileInputStream(file);

    try {
        parser.parse(stream, file.getAbsolutePath());
    } catch (Exception e) {
        throw new SailConfigException("Error parsing file!");
    }

    stream.close();
    return model;
}
 

protected void buildIndex(Pair<RocksDB, Map<COLUMN_FAMILIES, ColumnFamilyHandle>> dbAndHandles, List<File> rdfFiles)
    throws RocksDBException, ClassNotFoundException, IOException {
  LOGGER.info("Building RocksDB index of data in RDF files");
  RDFParser parser = Rio.createParser(RDFFormat.TURTLE);

  LOGGER.info("Processing %d RDF files", rdfFiles.size());
  for (File rdfFile : rdfFiles) {
    LOGGER.info("Processing file %s", rdfFile.getAbsolutePath());
    AbstractRDFHandler handler = PC_RDF_DATA_FILE_CONFIG.makeHandlerForDataFile(dbAndHandles, rdfFile);
    if (handler == null) {
      LOGGER.info("Skipping file without defined handler: %s", rdfFile.getAbsolutePath());
      continue;
    }

    parser.setRDFHandler(handler);
    parser.parse(new GZIPInputStream(new FileInputStream(rdfFile)), "");
    LOGGER.info("Successfully parsed file at %s", rdfFile.getAbsolutePath());
  }
  LOGGER.info("Done processing RDF files");
}
 

private static int getTriplesCount(String uri, String compression, RDFFormat format) throws Exception {
    InputStream in = FileSystem.get(URI.create(uri), HBaseServerTestInstance.getInstanceConfig()).open(new Path(uri));
    try {
        if (compression != null) {
            in = new CompressorStreamFactory().createCompressorInputStream(compression, in);
        }
        RDFParser parser = Rio.createParser(format);
        final AtomicInteger i = new AtomicInteger();
        parser.setRDFHandler(new AbstractRDFHandler(){
            @Override
            public void handleStatement(Statement st) throws RDFHandlerException {
                i.incrementAndGet();
            }
        });
        parser.parse(in, uri);
        return i.get();
    } finally {
        in.close();
    }
}
 

/***
 * Add a new repository to the manager.
 *
 * @param configStream input stream of the repository configuration
 * @return ID of the repository as string
 * @throws IOException
 * @throws RDF4JException
 */
protected String addRepository(InputStream configStream) throws IOException, RDF4JException {
	RDFParser rdfParser = Rio.createParser(RDFFormat.TURTLE, SimpleValueFactory.getInstance());

	Model graph = new LinkedHashModel();
	rdfParser.setRDFHandler(new StatementCollector(graph));
	rdfParser.parse(
			new StringReader(IOUtil.readString(new InputStreamReader(configStream, StandardCharsets.UTF_8))),
			RepositoryConfigSchema.NAMESPACE);
	configStream.close();

	Resource repositoryNode = Models.subject(graph.filter(null, RDF.TYPE, RepositoryConfigSchema.REPOSITORY))
			.orElseThrow(() -> new RepositoryConfigException("could not find subject resource"));

	RepositoryConfig repoConfig = RepositoryConfig.create(graph, repositoryNode);
	repoConfig.validate();
	manager.addRepositoryConfig(repoConfig);

	String repId = Models.objectLiteral(graph.filter(repositoryNode, RepositoryConfigSchema.REPOSITORYID, null))
			.orElseThrow(() -> new RepositoryConfigException("missing repository id"))
			.stringValue();

	return repId;
}
 

/**
 * Adds the data that can be read from the supplied InputStream or Reader to this repository.
 *
 * @param inputStreamOrReader An {@link InputStream} or {@link Reader} containing RDF data that must be added to the
 *                            repository.
 * @param baseURI             The base URI for the data.
 * @param dataFormat          The file format of the data.
 * @param rdfHandler          handles all data from all documents
 * @throws IOException
 * @throws UnsupportedRDFormatException
 * @throws RDFParseException
 * @throws RDFHandlerException
 */
private void loadInputStreamOrReader(Object inputStreamOrReader, String baseURI, RDFFormat dataFormat,
		RDFHandler rdfHandler) throws IOException, RDFParseException, RDFHandlerException {
	RDFParser rdfParser = Rio.createParser(dataFormat, vf);
	rdfParser.setParserConfig(config);
	rdfParser.setParseErrorListener(new ParseErrorLogger());

	rdfParser.setRDFHandler(rdfHandler);

	if (inputStreamOrReader instanceof InputStream) {
		rdfParser.parse((InputStream) inputStreamOrReader, baseURI);
	} else if (inputStreamOrReader instanceof Reader) {
		rdfParser.parse((Reader) inputStreamOrReader, baseURI);
	} else {
		throw new IllegalArgumentException(
				"Must be an InputStream or a Reader, is a: " + inputStreamOrReader.getClass());
	}
}
 

private void importInputStream(RepositoryConnection conn, ImportServiceConfig config, @Nonnull InputStream stream,
                               @Nonnull RDFFormat format) throws IOException {
    RDFParser parser = Rio.createParser(format);
    ParserConfig parserConfig = new ParserConfig();
    if (config.getContinueOnError()) {
        parserConfig.addNonFatalError(BasicParserSettings.FAIL_ON_UNKNOWN_DATATYPES);
        parserConfig.addNonFatalError(BasicParserSettings.FAIL_ON_UNKNOWN_LANGUAGES);
        parserConfig.addNonFatalError(BasicParserSettings.NORMALIZE_DATATYPE_VALUES);
    }
    parserConfig.addNonFatalError(BasicParserSettings.VERIFY_URI_SYNTAX);
    parser.setParserConfig(parserConfig);
    BatchInserter inserter = new BatchInserter(conn, transformer, config.getBatchSize());
    if (config.getLogOutput()) {
        inserter.setLogger(LOGGER);
    }
    if (config.getPrintOutput()) {
        inserter.setPrintToSystem(true);
    }
    parser.setRDFHandler(inserter);
    parser.parse(stream, "");
}
 

@Test
public void testStatsTargetPartial() throws Exception {
    final HBaseSail sail = new HBaseSail(HBaseServerTestInstance.getInstanceConfig(), "statsTable3", true, -1, true, 0, null, null);
    sail.initialize();
    try (InputStream ref = HalyardStatsTest.class.getResourceAsStream("testData.trig")) {
        RDFParser p = Rio.createParser(RDFFormat.TRIG);
        p.setPreserveBNodeIDs(true);
        p.setRDFHandler(new AbstractRDFHandler() {
            @Override
            public void handleStatement(Statement st) throws RDFHandlerException {
                sail.addStatement(st.getSubject(), st.getPredicate(), st.getObject(), st.getContext());
            }
        }).parse(ref, "");
    }
    sail.commit();
    sail.close();

    File root = File.createTempFile("test_stats", "");
    root.delete();
    root.mkdirs();

    assertEquals(0, ToolRunner.run(HBaseServerTestInstance.getInstanceConfig(), new HalyardStats(),
            new String[]{"-s", "statsTable3", "-t", root.toURI().toURL().toString() + "stats{0}.trig", "-r", "100", "-g", "http://whatever/myStats", "-c", "http://whatever/graph0"}));

    File stats = new File(root, "stats0.trig");
    assertTrue(stats.isFile());
    try (InputStream statsStream = new FileInputStream(stats)) {
        try (InputStream refStream = HalyardStatsTest.class.getResourceAsStream("testStatsTargetPartial.trig")) {
            Model statsM = Rio.parse(statsStream, "", RDFFormat.TRIG, new ParserConfig().set(BasicParserSettings.PRESERVE_BNODE_IDS, true), SimpleValueFactory.getInstance(), new ParseErrorLogger());
            Model refM = Rio.parse(refStream, "", RDFFormat.TRIG, new ParserConfig().set(BasicParserSettings.PRESERVE_BNODE_IDS, true), SimpleValueFactory.getInstance(), new ParseErrorLogger(), SimpleValueFactory.getInstance().createIRI("http://whatever/myStats"));
            assertEqualModels(refM, statsM);
        }
    }
}
 

@Before
public void setUp() throws Exception {
	parser = Rio.createParser(RDFFormat.RDFJSON);
	errors = new ParseErrorCollector();
	model = new LinkedHashModel();
	parser.setParseErrorListener(errors);
	parser.setRDFHandler(new ContextStatementCollector(model, SimpleValueFactory.getInstance()));
}
 

@Test
    public void testCreateFromTemplateName() throws Exception {
        LocalRepositoryManager repoman = new LocalRepositoryManager(Files.createTempDir());
        repoman.initialize();

        try(InputStream templateStream = RepositoryConfig.class.getResourceAsStream("RyaAccumuloSail.ttl")) {
            String template = IOUtils.toString(templateStream);

            final ConfigTemplate configTemplate = new ConfigTemplate(template);
            final Map<String, String> valueMap = ImmutableMap.<String, String> builder()
                    .put("Repository ID", "RyaAccumuloSail")
                    .put("Repository title", "RyaAccumuloSail Store")
                    .put("Rya Accumulo user", "root")
                    .put("Rya Accumulo password", "")
                    .put("Rya Accumulo instance", "dev")
                    .put("Rya Accumulo zookeepers", "zoo1,zoo2,zoo3")
                    .put("Rya Accumulo is mock", "true")
                    .build();

            final String configString = configTemplate.render(valueMap);

//            final Repository systemRepo = this.state.getManager().getSystemRepository();
            final Model model = new LinkedHashModel();
            final RDFParser rdfParser = Rio.createParser(RDFFormat.TURTLE);
            rdfParser.setRDFHandler(new StatementCollector(model));
            rdfParser.parse(new StringReader(configString), RepositoryConfigSchema.NAMESPACE);
            final Resource repositoryNode = Models.subject(model.filter(null, RDF.TYPE, RepositoryConfigSchema.REPOSITORY)).get();
            final RepositoryConfig repConfig = RepositoryConfig.create(model, repositoryNode);
            repConfig.validate();


            repoman.addRepositoryConfig(repConfig);

            Repository r = repoman.getRepository("RyaAccumuloSail");
            r.initialize();

        }

    }
 

/**
 * @throws java.lang.Exception
 */
@Before
public void setUp() throws Exception {
	vf = SimpleValueFactory.getInstance();
	settingsNoVerifyLangTag = new ParserConfig();
	settingsNoVerifyLangTag.set(BasicParserSettings.VERIFY_LANGUAGE_TAGS, false);
	errors = new ParseErrorCollector();
	parser = Rio.createParser(RDFFormat.TRIG);
	statementCollector = new StatementCollector(new LinkedHashModel());
	parser.setRDFHandler(statementCollector);
}
 

@Before
public void setUp() throws Exception {
	parser = Rio.createParser(RDFFormat.JSONLD);
	errors = new ParseErrorCollector();
	model = new LinkedHashModel();
	parser.setParseErrorListener(errors);
	parser.setRDFHandler(new ContextStatementCollector(model, F));
}
 

private void assertStatements(String ttl)
		throws RDFParseException, RDFHandlerException, IOException, RepositoryException {
	StatementCollector expected = new StatementCollector();
	RDFParser parser = Rio.createParser(RDFFormat.TURTLE);
	parser.setRDFHandler(expected);
	URL url = getClass().getResource(BASE_DIR + ttl);
	try (InputStream rdfStream = url.openStream()) {
		parser.parse(rdfStream, url.toString());
	}

	for (Statement stmt : expected.getStatements()) {
		assertTrue("Expected statement: " + stmt, conn.hasStatement(stmt, true));
	}
}
 

@Test
public void testSpinRenderer() throws IOException, RDF4JException {
	StatementCollector expected = new StatementCollector();
	RDFParser parser = Rio.createParser(RDFFormat.TURTLE);
	parser.setRDFHandler(expected);
	try (InputStream rdfStream = testURL.openStream()) {
		parser.parse(rdfStream, testURL.toString());
	}

	// get query from sp:text
	String query = null;
	for (Statement stmt : expected.getStatements()) {
		if (SP.TEXT_PROPERTY.equals(stmt.getPredicate())) {
			query = stmt.getObject().stringValue();
			break;
		}
	}
	assertNotNull(query);

	ParsedOperation parsedOp = QueryParserUtil.parseOperation(QueryLanguage.SPARQL, query, testURL.toString());

	StatementCollector actual = new StatementCollector();
	renderer.render(parsedOp, actual);

	Object operation = (parsedOp instanceof ParsedQuery) ? ((ParsedQuery) parsedOp).getTupleExpr()
			: ((ParsedUpdate) parsedOp).getUpdateExprs();
	assertTrue("Operation was\n" + operation + "\nExpected\n" + toRDF(expected) + "\nbut was\n" + toRDF(actual),
			Models.isomorphic(actual.getStatements(), expected.getStatements()));
}
 

@Test
public void testSpinParser() throws IOException, RDF4JException {
	StatementCollector expected = new StatementCollector();
	RDFParser parser = Rio.createParser(RDFFormat.TURTLE);
	parser.setRDFHandler(expected);
	try (InputStream rdfStream = testURL.openStream()) {
		parser.parse(rdfStream, testURL.toString());
	}

	// get query resource from sp:text
	Resource queryResource = null;
	for (Statement stmt : expected.getStatements()) {
		if (SP.TEXT_PROPERTY.equals(stmt.getPredicate())) {
			queryResource = stmt.getSubject();
			break;
		}
	}
	assertNotNull(queryResource);

	TripleSource store = new ModelTripleSource(new TreeModel(expected.getStatements()));
	ParsedOperation textParsedOp = textParser.parse(queryResource, store);
	ParsedOperation rdfParsedOp = rdfParser.parse(queryResource, store);

	if (textParsedOp instanceof ParsedQuery) {
		assertEquals(((ParsedQuery) textParsedOp).getTupleExpr(), ((ParsedQuery) rdfParsedOp).getTupleExpr());
	} else {
		assertEquals(((ParsedUpdate) textParsedOp).getUpdateExprs(), ((ParsedUpdate) rdfParsedOp).getUpdateExprs());
	}
}
 

RepositoryConfig updateRepositoryConfig(final String configString) throws IOException, RDF4JException {
	final Model graph = new LinkedHashModel();
	final RDFParser rdfParser = Rio.createParser(RDFFormat.TURTLE, SimpleValueFactory.getInstance());
	rdfParser.setRDFHandler(new StatementCollector(graph));
	rdfParser.parse(new StringReader(configString), RepositoryConfigSchema.NAMESPACE);

	Resource res = Models.subject(graph.getStatements(null, RDF.TYPE, RepositoryConfigSchema.REPOSITORY))
			.orElseThrow(() -> new RepositoryException("could not find instance of Repository class in config"));
	final RepositoryConfig repConfig = RepositoryConfig.create(graph, res);
	repConfig.validate();
	manager.addRepositoryConfig(repConfig);
	return repConfig;
}
 

private void upload(IRI graphURI, Resource context) throws Exception {
	RepositoryConnection con = dataRep.getConnection();

	try {
		con.begin();
		RDFFormat rdfFormat = Rio.getParserFormatForFileName(graphURI.toString()).orElse(RDFFormat.TURTLE);
		RDFParser rdfParser = Rio.createParser(rdfFormat, dataRep.getValueFactory());
		rdfParser.setVerifyData(false);
		rdfParser.setDatatypeHandling(DatatypeHandling.IGNORE);
		// rdfParser.setPreserveBNodeIDs(true);

		RDFInserter rdfInserter = new RDFInserter(con);
		rdfInserter.enforceContext(context);
		rdfParser.setRDFHandler(rdfInserter);

		URL graphURL = new URL(graphURI.toString());
		InputStream in = graphURL.openStream();
		try {
			rdfParser.parse(in, graphURI.toString());
		} finally {
			in.close();
		}

		con.commit();
	} catch (Exception e) {
		if (con.isActive()) {
			con.rollback();
		}
		throw e;
	} finally {
		con.close();
	}
}
 

protected void upload(IRI graphURI, Resource context) throws Exception {
	RepositoryConnection con = getDataRepository().getConnection();

	try {
		con.begin();
		RDFFormat rdfFormat = Rio.getParserFormatForFileName(graphURI.toString()).orElse(RDFFormat.TURTLE);
		RDFParser rdfParser = Rio.createParser(rdfFormat, getDataRepository().getValueFactory());
		rdfParser.setVerifyData(false);
		rdfParser.setDatatypeHandling(DatatypeHandling.IGNORE);
		// rdfParser.setPreserveBNodeIDs(true);

		RDFInserter rdfInserter = new RDFInserter(con);
		rdfInserter.enforceContext(context);
		rdfParser.setRDFHandler(rdfInserter);

		URL graphURL = new URL(graphURI.toString());
		InputStream in = graphURL.openStream();
		try {
			rdfParser.parse(in, graphURI.toString());
		} finally {
			in.close();
		}

		con.commit();
	} catch (Exception e) {
		if (con.isActive()) {
			con.rollback();
		}
		throw e;
	} finally {
		con.close();
	}
}
 

/**
 * Parse the response in a background thread. HTTP connections are dealt with in the {@link BackgroundGraphResult}
 * or (in the error-case) in this method.
 */
protected GraphQueryResult getRDFBackground(HttpUriRequest method, boolean requireContext)
		throws IOException, RDFHandlerException, RepositoryException, MalformedQueryException,
		UnauthorizedException, QueryInterruptedException {

	boolean submitted = false;

	// Specify which formats we support using Accept headers
	Set<RDFFormat> rdfFormats = RDFParserRegistry.getInstance().getKeys();
	if (rdfFormats.isEmpty()) {
		throw new RepositoryException("No tuple RDF parsers have been registered");
	}

	GraphQueryResult gRes = null;
	// send the tuple query
	HttpResponse response = sendGraphQueryViaHttp(method, requireContext, rdfFormats);
	try {

		// if we get here, HTTP code is 200
		String mimeType = getResponseMIMEType(response);
		RDFFormat format = RDFFormat.matchMIMEType(mimeType, rdfFormats)
				.orElseThrow(() -> new RepositoryException(
						"Server responded with an unsupported file format: " + mimeType));
		RDFParser parser = Rio.createParser(format, getValueFactory());
		parser.setParserConfig(getParserConfig());
		parser.setParseErrorListener(new ParseErrorLogger());

		Charset charset = null;

		// SES-1793 : Do not attempt to check for a charset if the format is
		// defined not to have a charset
		// This prevents errors caused by people erroneously attaching a
		// charset to a binary formatted document
		HttpEntity entity = response.getEntity();
		if (format.hasCharset() && entity != null && entity.getContentType() != null) {
			// TODO copied from SPARQLGraphQuery repository, is this
			// required?
			try {
				charset = ContentType.parse(entity.getContentType().getValue()).getCharset();
			} catch (IllegalCharsetNameException e) {
				// work around for Joseki-3.2
				// Content-Type: application/rdf+xml;
				// charset=application/rdf+xml
			}
			if (charset == null) {
				charset = UTF8;
			}
		}

		if (entity == null) {
			throw new RepositoryException("Server response was empty.");
		}

		String baseURI = method.getURI().toASCIIString();
		gRes = background.parse(parser, entity.getContent(), charset, baseURI);
		submitted = true;
		return gRes;
	} finally {
		if (!submitted) {
			EntityUtils.consumeQuietly(response.getEntity());
		}
	}

}
 

@Override
public void fromFile(final Path statementsPath, final String visibilities) throws RyaStreamsException {
    requireNonNull(statementsPath);
    requireNonNull(visibilities);

    if(!statementsPath.toFile().exists()) {
        throw new RyaStreamsException("Could not load statements at path '" + statementsPath + "' because that " +
                "does not exist. Make sure you've entered the correct path.");
    }

    // Create an RDF Parser whose format is derived from the statementPath's file extension.
    final String filename = statementsPath.getFileName().toString();
    final RDFFormat format = RdfFormatUtils.forFileName(filename);
    if (format == null) {
        throw new UnsupportedRDFormatException("Unknown RDF format for the file: " + filename);
    }
    final RDFParser parser = Rio.createParser(format);

    // Set a handler that writes the statements to the specified kafka topic.
    parser.setRDFHandler(new AbstractRDFHandler() {
        @Override
        public void startRDF() throws RDFHandlerException {
            log.trace("Starting loading statements.");
        }

        @Override
        public void handleStatement(final Statement stmnt) throws RDFHandlerException {
            final VisibilityStatement visiStatement = new VisibilityStatement(stmnt, visibilities);
            producer.send(new ProducerRecord<>(topic, visiStatement));
        }

        @Override
        public void endRDF() throws RDFHandlerException {
            producer.flush();
            log.trace("Done.");
        }
    });

    // Do the parse and load.
    try {
        parser.parse(Files.newInputStream(statementsPath), "");
    } catch (RDFParseException | RDFHandlerException | IOException e) {
        throw new RyaStreamsException("Could not load the RDF file's Statements into Rya Streams.", e);
    }
}
 

/**
 * Instantiates the RecordReader.
 * @param format    RDF serialization format to parse.
 * @param charBufferSize    Number of input characters to hold in
 *                          memory; if exceeded, wait until the parser
 *                          thread consumes some text before proceeding
 *                          with reading input.
 * @param statementBufferSize   Number of output statements to hold in
 *                              memory; if exceeded, wait until the
 *                              client consumes data before proceeding
 *                              with parsing.
 * @param timeoutSeconds    Number of seconds to wait for the parser
 *                          thread to provide the next statement (or
 *                          state that there are none). If exceeded,
 *                          abort.
 */
RdfFileRecordReader(RDFFormat format, int charBufferSize, int statementBufferSize, int timeoutSeconds) {
    rdfParser = Rio.createParser(format);
    rdfParser.setRDFHandler(this);
    statementCache = new LinkedBlockingQueue<RyaStatementWritable>(statementBufferSize);
    pipeOut = new PipedWriter();
    pipeIn = new PipedReader(charBufferSize);
    this.timeoutSeconds = timeoutSeconds;
    logger.info("Initializing RecordReader with parameters:");
    logger.info("\tRDF serialization format = " + format.getName());
    logger.info("\tinput buffer size = " + charBufferSize + " characters");
    logger.info("\tstatement cache size = " + statementBufferSize);
    logger.info("\tparser timeout = " + timeoutSeconds + " seconds");
}