org.apache.jena.riot.tokens.TokenType Java Examples

private void readPrefixedNameOrKeyword(Token token) {
    long posn = reader.getPosition();
    String prefixPart = readPrefixPart(); // Prefix part or keyword
    token.setImage(prefixPart);
    token.setType(TokenType.KEYWORD);
    int ch = reader.peekChar();
    if ( ch == CH_COLON ) {
        reader.readChar();
        token.setType(TokenType.PREFIXED_NAME);
        String ln = readLocalPart(); // Local part
        token.setImage2(ln);
        if ( Checking )
            checkPrefixedName(token.getImage(), token.getImage2());
    }

    // If we made no progress, nothing found, not even a keyword -- it's an
    // error.
    if ( posn == reader.getPosition() )
        error("Failed to find a prefix name or keyword: %c(%d;0x%04X)", ch, ch, ch);

    if ( Checking )
        checkKeyword(token.getImage());

}
 

@Override
public Node create(Node currentGraph, Token token) {
  int line = (int) token.getLine();
  int col = (int) token.getColumn();
  String str = token.getImage();
  validationModels.forEach(m -> {
    Node node = null;
    int length = 0;
    if (token.getType() == TokenType.PREFIXED_NAME) {
      String prefix = str;
      String suffix = token.getImage2();
      String expansion = expandPrefixedName(prefix, suffix);
      node = createURI(expansion, line, col);
      length = str.length() + suffix.length();
    } else if (token.getType() == TokenType.IRI) {
      node = createURI(str, line, col);
      length = str.length();
    }
    if (node != null) {
      List<LintProblem> diagnostic = m
          .validateNode(node, line, col, line, col + length);
      diagnosticList.addAll(diagnostic);
    }
  });
  return super.create(currentGraph, token);
}
 

@Test
public void embeddedTripleOK() {
	final Token token = getToken( "<<:s :p :o>>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	assertEquals(TokenType.PREFIXED_NAME, tokenS.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenO.getType());
}
 

@Test
public void nesting1() {
	final Token token = getToken( "<< <<:s :p :o>> :p2 :o2>>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	confirmEmbeddedTripleToken(tokenS);
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenO.getType());
}
 

@Test
public void nesting2() {
	final Token token = getToken( "<<<<:s :p :o>> :p2 :o2>>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	confirmEmbeddedTripleToken(tokenS);
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenO.getType());
}
 

@Test
public void nesting3() {
	final Token token = getToken( "<< :s2 :p2 <<:s :p :o>> >>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	assertEquals(TokenType.PREFIXED_NAME, tokenS.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	confirmEmbeddedTripleToken(tokenO);
}
 

@Test
public void nesting4() {
	final Token token = getToken( "<< :s2 :p2 <<:s :p :o>>>>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	assertEquals(TokenType.PREFIXED_NAME, tokenS.getType());
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	confirmEmbeddedTripleToken(tokenO);
}
 

@Test
public void nesting5() {
	final Token token = getToken( "<<<<:s :p :o>> :p2 <<:s :p :o>>>>" );

	confirmEmbeddedTripleToken(token);

	final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
	final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
	final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);

	confirmEmbeddedTripleToken(tokenS);
	assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
	confirmEmbeddedTripleToken(tokenO);
}
 

private void readNumber() {
        // One entry, definitely a number.
        // Beware of '.' as a (non) decimal.
        /*
        maybeSign()
        digits()
        if dot ==> decimal, digits
        if e   ==> double, maybeSign, digits
        else
            check not "." for decimal.
        */
        boolean isDouble = false;
        boolean isDecimal = false;
        stringBuilder.setLength(0);
        
        /*
        readPossibleSign(stringBuilder);
        readDigits may be hex
        readDot
        readDigits
        readExponent.
        */
        
        int x = 0; // Digits before a dot.
        int ch = reader.peekChar();
        if ( ch == '0' ) {
            x++;
            reader.readChar();
            stringBuilder.append((char)ch);
            ch = reader.peekChar();
            if ( ch == 'x' || ch == 'X' ) {
                reader.readChar();
                stringBuilder.append((char)ch);
                readHex(reader, stringBuilder);
                token.setImage(stringBuilder.toString());
                token.setType(TokenType.HEX);
                return;
            }
        } else if ( ch == '-' || ch == '+' ) {
            readPossibleSign(stringBuilder);
        }

        x += readDigits(stringBuilder);
//        if ( x == 0 ) {}
        ch = reader.peekChar();
        if ( ch == CH_DOT ) {
            reader.readChar();
            stringBuilder.append(CH_DOT);
            isDecimal = true; // Includes things that will be doubles.
            readDigits(stringBuilder);
        }

        if ( x == 0 && !isDecimal )
            // Possible a tokenizer error - should not have entered readNumber
            // in the first place.
            error("Unrecognized as number");

        if ( exponent(stringBuilder) ) {
            isDouble = true;
            isDecimal = false;
        }

        // Final part - "decimal" 123. is an integer 123 and a DOT.
        if ( isDecimal ) {
            int len = stringBuilder.length();
            if ( stringBuilder.charAt(len - 1) == CH_DOT ) {
                stringBuilder.setLength(len - 1);
                reader.pushbackChar(CH_DOT);
                isDecimal = false;
            }
        }

        token.setImage(stringBuilder.toString());
        if ( isDouble )
            token.setType(TokenType.DOUBLE);
        else if ( isDecimal )
            token.setType(TokenType.DECIMAL);
        else
            token.setType(TokenType.INTEGER);
    }
 

private static void skip(Tokenizer tokenizer, TokenType tokenType ) {
    Token tok = tokenizer.next();
    if ( ! tok.hasType(tokenType) )
        throw exception(tok, "Expected token type: "+tokenType+": got "+tok);
}