org.apache.jena.riot.tokens.Token Java Examples

/**
 * Parse a string into a stream of Quads.
 * @param line the line of text
 * @return the Quad
 */
public static Stream<Quad> parseQuad(final String line) {
    final List<Token> tokens = new ArrayList<>();
    makeTokenizerString(line).forEachRemaining(tokens::add);

    final List<Node> nodes = tokens.stream().filter(Token::isNode).map(Token::asNode).filter(Objects::nonNull)
        .collect(toList());

    if (nodes.size() == 3) {
        return of(fromJena(create(defaultGraphIRI, nodes.get(0), nodes.get(1), nodes.get(2))));
    } else if (nodes.size() == 4) {
        return of(fromJena(create(nodes.get(3), nodes.get(0), nodes.get(1), nodes.get(2))));
    } else {
        LOGGER.warn("Skipping invalid data value: {}", line);
        return empty();
    }
}
 

private void readPrefixedNameOrKeyword(Token token) {
    long posn = reader.getPosition();
    String prefixPart = readPrefixPart(); // Prefix part or keyword
    token.setImage(prefixPart);
    token.setType(TokenType.KEYWORD);
    int ch = reader.peekChar();
    if ( ch == CH_COLON ) {
        reader.readChar();
        token.setType(TokenType.PREFIXED_NAME);
        String ln = readLocalPart(); // Local part
        token.setImage2(ln);
        if ( Checking )
            checkPrefixedName(token.getImage(), token.getImage2());
    }

    // If we made no progress, nothing found, not even a keyword -- it's an
    // error.
    if ( posn == reader.getPosition() )
        error("Failed to find a prefix name or keyword: %c(%d;0x%04X)", ch, ch, ch);

    if ( Checking )
        checkKeyword(token.getImage());

}
 

@Override
public Node create(Node currentGraph, Token token) {
  int line = (int) token.getLine();
  int col = (int) token.getColumn();
  String str = token.getImage();
  validationModels.forEach(m -> {
    Node node = null;
    int length = 0;
    if (token.getType() == TokenType.PREFIXED_NAME) {
      String prefix = str;
      String suffix = token.getImage2();
      String expansion = expandPrefixedName(prefix, suffix);
      node = createURI(expansion, line, col);
      length = str.length() + suffix.length();
    } else if (token.getType() == TokenType.IRI) {
      node = createURI(str, line, col);
      length = str.length();
    }
    if (node != null) {
      List<LintProblem> diagnostic = m
          .validateNode(node, line, col, line, col + length);
      diagnosticList.addAll(diagnostic);
    }
  });
  return super.create(currentGraph, token);
}
 

/** Read patch header. */
public static PatchHeader readerHeader(InputStream input) {
    Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input);
    Map<String, Node> header = new LinkedHashMap<>();
    int lineNumber = 0;
    while(tokenizer.hasNext()) {
      Token tokCode = tokenizer.next();
      if ( tokCode.hasType(DOT) )
          throw exception(tokCode, "Empty header line");
      if ( ! tokCode.isWord() )
          throw exception(tokCode, "Expected keyword at start of patch header");
      String code = tokCode.getImage();
      lineNumber ++;
      if ( ! code.equals(PatchCodes.HEADER) )
          break;
      readHeaderLine(tokenizer, (f,n)->header.put(f, n));
  }
  return new PatchHeader(header);
}
 

@Override
public final Token next() {
    if ( !hasNext() )
        throw new NoSuchElementException();
    Token t = token;
    token = null;
    return t;
}
 

protected Node getNode( final String string ) {
	final Tokenizer tokenizer = TokenizerFactory.makeTokenizerString(string);
	final Token token = tokenizer.next();
	tokenizer.close();

       return parserProfile.create(null, token);
}
 

protected void confirmEmbeddedTripleToken( final Token token ) {

		assertEquals(EmbeddedTripleTokenUtils.typeOfEmbeddedTripleTokens, token.getType());
		assertEquals(EmbeddedTripleTokenUtils.image1OfEmbeddedTripleTokens, token.getImage());
		assertEquals(EmbeddedTripleTokenUtils.image2OfEmbeddedTripleTokens, token.getImage2());

		assertNotNull( token.getSubToken1() );
		assertNotNull( token.getSubToken2() );
		assertNotNull( token.getSubToken2().getSubToken1() );
		assertNotNull( token.getSubToken2().getSubToken2() );

		assertTrue( EmbeddedTripleTokenUtils.isEmbeddedTripleToken(token) );
	}
 

protected Token getToken( final String string ) {
	final PeekReader reader = PeekReader.readString(string);
	final TokenizerText tokenizer = new TokenizerText(reader);
	final Token token = tokenizer.next();
	tokenizer.close();

	return token;
}
 

@Test
public void nesting5() {
	final Token token = getToken( "<<<<:s :p :o>> :p2 <<:s :p :o>>>>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	confirmEmbeddedTripleToken(tokenS);
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	confirmEmbeddedTripleToken(tokenO);
}
 

@Test
public void nesting4() {
	final Token token = getToken( "<< :s2 :p2 <<:s :p :o>>>>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	assertEquals(TokenType.PREFIXED_NAME, tokenS.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	confirmEmbeddedTripleToken(tokenO);
}
 

@Test
public void nesting3() {
	final Token token = getToken( "<< :s2 :p2 <<:s :p :o>> >>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	assertEquals(TokenType.PREFIXED_NAME, tokenS.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	confirmEmbeddedTripleToken(tokenO);
}
 

@Test
public void nesting2() {
	final Token token = getToken( "<<<<:s :p :o>> :p2 :o2>>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	confirmEmbeddedTripleToken(tokenS);
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenO.getType());
}
 

@Test
public void nesting1() {
	final Token token = getToken( "<< <<:s :p :o>> :p2 :o2>>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	confirmEmbeddedTripleToken(tokenS);
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenO.getType());
}
 

@Test
public void embeddedTripleOK() {
	final Token token = getToken( "<<:s :p :o>>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	assertEquals(TokenType.PREFIXED_NAME, tokenS.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenO.getType());
}
 

@Test
public void failTooManyElements() {
	try {
		Token token = getToken( "<<:s :p :o :o>>" );
		fail("Expected exception missing");

		if ( token == null ) token = null; // avoid compiler warning
	}
	catch ( RiotParseException e ) {
		// this is expected
	}
}
 

@Test
public void failTooFewElements3() {
	try {
		Token token = getToken( "<<>>" );
		fail("Expected exception missing");

		if ( token == null ) token = null; // avoid compiler warning
	}
	catch ( RiotParseException e ) {
		// this is expected
	}
}
 

@Test
public void failTooFewElements2() {
	try {
		Token token = getToken( "<<:s >>" );
		fail("Expected exception missing");

		if ( token == null ) token = null; // avoid compiler warning
	}
	catch ( RiotParseException e ) {
		// this is expected
	}
}
 

@Test
public void failTooFewElements1() {
	try {
		Token token = getToken( "<<:s :p >>" );
		fail("Expected exception missing");

		if ( token == null ) token = null; // avoid compiler warning
	}
	catch ( RiotParseException e ) {
		// this is expected
	}
}
 

@Override
public Node createNodeFromToken(Node scope, Token token, long line, long col) {
	if ( EmbeddedTripleTokenUtils.isEmbeddedTripleToken(token) )
		return ParserProfileTurtleStar.Helper.createTripleNodeFromEmbeddedTripleToken(this, scope, token, line, col);
	else
		return super.createNodeFromToken(scope, token, line, col);
}
 

@Override
public Node create(Node currentGraph, Token token) {
	if ( EmbeddedTripleTokenUtils.isEmbeddedTripleToken(token) )
		return ParserProfileTurtleStar.Helper.createTripleNodeFromEmbeddedTripleToken(
				      this, currentGraph, token, token.getLine(), token.getColumn());
	else
		return profile.create(currentGraph, token);
}
 

@Override
public Node createNodeFromToken(Node scope, Token token, long line, long col) {
	if ( EmbeddedTripleTokenUtils.isEmbeddedTripleToken(token) )
		return ParserProfileTurtleStar.Helper.createTripleNodeFromEmbeddedTripleToken(this, scope, token, line, col);
	else
		return profile.createNodeFromToken(scope, token, line, col);
}
 

static public Node createTripleNodeFromEmbeddedTripleToken(
		final ParserProfile profile, final Node scope,
		final Token token, final long line, final long col) 
{
	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	final Node s = profile.create(scope, tokenS);
	final Node p = profile.create(scope, tokenP);
	final Node o = profile.create(scope, tokenO);

	final Triple t = profile.createTriple(s, p, o, token.getLine(), token.getColumn());
	return new Node_Triple(t);
}
 

private static Node tokenToNode(Token token) {
    if ( token.isIRI() )
        // URI or <_:...>
        return RiotLib.createIRIorBNode(token.getImage());
    if ( token.isBNode() ) {
        // Blank node as _:...
        String label = token.getImage().substring(bNodeLabelStart.length());
        return NodeFactory.createBlankNode(label);
    }
    Node node = token.asNode();
    if ( node == null )
        throw exception(token, "Expect a Node, got %s",token);
    return node;
}
 

/** Known-to-be-header line */
private static void readHeaderLine(Tokenizer tokenizer, BiConsumer<String, Node> action) {
    Token token2 = nextToken(tokenizer);
    if ( ! token2.isWord() && ! token2.isString() )
        throw exception(tokenizer, "Header does not have a key that is a word: "+token2);
    String field = token2.getImage();
    Node v = nextNode(tokenizer);
    skip(tokenizer, DOT);
    action.accept(field, v);
}
 

static protected void setSubTokens( final Token t, final Token s, final Token p, final Token o ) {
   	final Token po = new Token(p.getLine(), p.getColumn());
   	po.setSubToken1(p);
   	po.setSubToken2(o);

   	t.setSubToken1(s);
   	t.setSubToken2(po);
}
 

private static Node nextNodeMaybe(Tokenizer tokenizer) {
    Token tok = tokenizer.peek();
    if ( tok.hasType(DOT) )
        return null;
    if ( tok.isEOF() )
        throw exception(tokenizer, "Input truncated: no DOT seen on last line");
    return tokenToNode(tokenizer.next());
}
 

private static Token nextToken(Tokenizer tokenizer) {
    if ( ! tokenizer.hasNext() )
        throw exception(tokenizer, "Input truncated");
    Token tok = tokenizer.next();
    if ( tok.hasType(DOT) )
        throw exception(tok, "Input truncated by DOT: line too short");
    if ( tok.isEOF() )
        throw exception(tok, "Input truncated: no DOT seen on last line");
    return tok;
}
 

private static void skip(Tokenizer tokenizer, TokenType tokenType ) {
    Token tok = tokenizer.next();
    if ( ! tok.hasType(tokenType) )
        throw exception(tok, "Expected token type: "+tokenType+": got "+tok);
}
 

private static PatchException exception(Token token, String fmt, Object... args) {
     String msg = String.format(fmt, args);
     if ( token != null )
         msg = SysRIOT.fmtMessage(msg, token.getLine(), token.getColumn());
     return new PatchException(msg);
}
 

@Override
public final Token peek() {
    if ( !hasNext() )
        return null;
    return token;
}