Java Code Examples for java.io.StreamTokenizer#TT_EOL

The following examples show how to use java.io.StreamTokenizer#TT_EOL . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileSourceBase.java    From gama with GNU General Public License v3.0 6 votes vote down vote up 
/**
 * Read a string or number or pushback and return null. If it is a number it is converted to a string before being
 * returned.
 */
protected String getStringOrWordOrNumberOrPushback() throws IOException {
	final int tok = st.nextToken();

	if (tok == StreamTokenizer.TT_EOL || tok == StreamTokenizer.TT_EOF) {
		pushBack();
		return null;
	}

	if (tok == StreamTokenizer.TT_NUMBER) {
		if (st.nval - (int) st.nval == 0) {
			return Integer.toString((int) st.nval);
		} else {
			return Double.toString(st.nval);
		}
	} else if (tok == StreamTokenizer.TT_WORD || tok == QUOTE_CHAR) {
		return st.sval;
	} else {
		pushBack();
		return null;
	}
}
Example 2
Source File: Harness.java    From openjdk-jdk8u with GNU General Public License v2.0 6 votes vote down vote up 
String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
Example 3
Source File: FileSourceBase.java    From gama with GNU General Public License v3.0 6 votes vote down vote up 
/**
 * Read a string or number or generate a parse error. If it is a number it is converted to a string before being
 * returned.
 */
protected String getStringOrWordOrNumber() throws IOException {
	final int tok = st.nextToken();

	if (tok == StreamTokenizer.TT_EOL || tok == StreamTokenizer.TT_EOF) {
		parseError("expecting word, string or number, " + gotWhat(tok));
	}

	if (tok == StreamTokenizer.TT_NUMBER) {
		if (st.nval - (int) st.nval == 0) {
			return Integer.toString((int) st.nval);
		} else {
			return Double.toString(st.nval);
		}
	} else {
		return st.sval;
	}
}
Example 4
Source File: ArffLoader.java    From samoa with Apache License 2.0 6 votes vote down vote up 
public Instance readInstance(Reader reader) {
    if (streamTokenizer == null) {
        initStreamTokenizer(reader);
    }
    while (streamTokenizer.ttype == StreamTokenizer.TT_EOL) {
        try {
            streamTokenizer.nextToken();
        } catch (IOException ex) {
            Logger.getLogger(ArffLoader.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    if (streamTokenizer.ttype == '{') {
        return readInstanceSparse();
        // return readDenseInstanceSparse();
    } else {
        return readInstanceDense();
    }

}
Example 5
Source File: Token.java    From hottub with GNU General Public License v2.0 6 votes vote down vote up 
public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
Example 6
Source File: Parser.java    From big-c with Apache License 2.0 6 votes vote down vote up 
Token next() throws IOException {
  int type = tok.nextToken();
  switch (type) {
    case StreamTokenizer.TT_EOF:
    case StreamTokenizer.TT_EOL:
      return null;
    case StreamTokenizer.TT_NUMBER:
      return new NumToken(tok.nval);
    case StreamTokenizer.TT_WORD:
      return new StrToken(TType.IDENT, tok.sval);
    case '"':
      return new StrToken(TType.QUOT, tok.sval);
    default:
      switch (type) {
        case ',':
          return new Token(TType.COMMA);
        case '(':
          return new Token(TType.LPAREN);
        case ')':
          return new Token(TType.RPAREN);
        default:
          throw new IOException("Unexpected: " + type);
      }
  }
}
Example 7
Source File: Token.java    From openjdk-jdk8u-backup with GNU General Public License v2.0 6 votes vote down vote up 
public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
Example 8
Source File: Token.java    From jdk8u60 with GNU General Public License v2.0 6 votes vote down vote up 
public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
Example 9
Source File: ArffLoader.java    From tsml with GNU General Public License v3.0 5 votes vote down vote up 
/**
 * Gets next token, checking for a premature and of line.
 *
 * @throws IOException 	if it finds a premature end of line
 */
protected void getNextToken() throws IOException {
  if (m_Tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
    errorMessage("premature end of line");
  }
  if (m_Tokenizer.ttype == StreamTokenizer.TT_EOF) {
    errorMessage("premature end of file");
  } else if ((m_Tokenizer.ttype == '\'') ||
      (m_Tokenizer.ttype == '"')) {
    m_Tokenizer.ttype = StreamTokenizer.TT_WORD;
  } else if ((m_Tokenizer.ttype == StreamTokenizer.TT_WORD) &&
      (m_Tokenizer.sval.equals("?"))){
    m_Tokenizer.ttype = '?';
  }
}
Example 10
Source File: ClassReaderState.java    From reladomo with Apache License 2.0 5 votes vote down vote up 
public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    int token = st.ttype;
    if (token != StreamTokenizer.TT_WORD || !st.sval.equals(AbstractMithraDataFileParser.CLASS_IDENTIFIER))
    {
        throw new ParseException("expected line " + st.lineno() + " to begin with class", st.lineno());
    }
    token = st.nextToken();
    if (token != StreamTokenizer.TT_WORD)
    {
        throw new ParseException("expected a class name on line "+st.lineno(), st.lineno());
    }
    this.getParser().addNewMithraParsedData();
    String className = st.sval;
    try
    {
        this.getParser().getCurrentParsedData().setParsedClassName(className);
    }
    catch (Exception e)
    {
        ParseException parseException = new ParseException("no such class (or finder): "+className+" on line "+st.lineno(), st.lineno());
        parseException.initCause(e);
        throw parseException;
    }
    this.getParser().getDataReaderState().setClass(className, st.lineno());

    token = st.nextToken();
    if (token != StreamTokenizer.TT_EOL)
    {
        throw new ParseException("invalid data after the class name on line "+st.lineno(), st.lineno());
    }

    return this.getParser().getAttributeReaderState();
}
Example 11
Source File: StreamTokenizerUtils.java    From tsml with GNU General Public License v3.0 5 votes vote down vote up 
/**
 * Gets token, skipping empty lines.
 * 
 * @param tokenizer the stream tokenizer
 * @throws IOException if reading the next token fails
 */
public static void getFirstToken(StreamTokenizer tokenizer)
    throws IOException {

  while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
  }
  ;
  if ((tokenizer.ttype == '\'') || (tokenizer.ttype == '"')) {
    tokenizer.ttype = StreamTokenizer.TT_WORD;
  } else if ((tokenizer.ttype == StreamTokenizer.TT_WORD)
      && (tokenizer.sval.equals("?"))) {
    tokenizer.ttype = '?';
  }
}
Example 12
Source File: FileSourceBase.java    From gama with GNU General Public License v3.0 5 votes vote down vote up 
/**
 * Get a symbol.
 */
protected char getSymbol() throws IOException {
	final int tok = st.nextToken();

	if (tok > 0 && tok != StreamTokenizer.TT_WORD && tok != StreamTokenizer.TT_NUMBER
			&& tok != StreamTokenizer.TT_EOL && tok != StreamTokenizer.TT_EOF && tok != QUOTE_CHAR
			&& tok != COMMENT_CHAR) {
		return (char) tok;
	}

	parseError("expecting a symbol, " + gotWhat(tok));
	return (char) 0; // Never reached.
}
Example 13
Source File: FileSourceBase.java    From gama with GNU General Public License v3.0 5 votes vote down vote up 
/**
 * Eat all availables EOLs.
 */
protected void eatAllEols() throws IOException {
	if (!eol_is_significant) { return; }

	int tok = st.nextToken();

	while (tok == StreamTokenizer.TT_EOL) {
		tok = st.nextToken();
	}

	pushBack();
}
Example 14
Source File: ReadAhead.java    From openjdk-jdk8u with GNU General Public License v2.0 5 votes vote down vote up 
private static void test(StreamTokenizer st) throws Exception {
    st.eolIsSignificant(true);
    int tt = st.nextToken();
    if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
    if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
    tt = st.nextToken();
    if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
Example 15
Source File: ReadAhead.java    From jdk8u_jdk with GNU General Public License v2.0 5 votes vote down vote up 
private static void test(StreamTokenizer st) throws Exception {
    st.eolIsSignificant(true);
    int tt = st.nextToken();
    if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
    if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
    tt = st.nextToken();
    if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
Example 16
Source File: AttributeReaderState.java    From reladomo with Apache License 2.0 5 votes vote down vote up 
private void parseAttributes(StreamTokenizer st) throws IOException, ParseException
{
    int token = st.nextToken();
    boolean wantAttribute = true;
    while(token != StreamTokenizer.TT_EOL)
    {
        if (wantAttribute)
        {
            if (token != StreamTokenizer.TT_WORD)
            {
                throw new ParseException("expected an attribute name on line "+st.lineno(), st.lineno());
            }
            this.getParser().getCurrentParsedData().addAttribute(st.sval, st.lineno());
        }
        else
        {
            if (token != ',')
            {
                throw new ParseException("Expected a comma on line "+st.lineno(), st.lineno());
            }
        }
        wantAttribute = !wantAttribute;
        token = st.nextToken();
    }
    if (wantAttribute)
    {
        throw new ParseException("extra comma at the end of line "+st.lineno(), st.lineno());
    }
}
Example 17
Source File: Harness.java    From openjdk-jdk9 with GNU General Public License v2.0 5 votes vote down vote up 
BenchInfo parseBenchInfo(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight = parseBenchWeight(tokens);
    String name = parseBenchName(tokens);
    Benchmark bench = parseBenchClass(tokens);
    String[] args = parseBenchArgs(tokens);
    if (tokens.ttype == StreamTokenizer.TT_EOL)
        tokens.nextToken();
    return new BenchInfo(bench, name, weight, args);
}
Example 18
Source File: Harness.java    From openjdk-8-source with GNU General Public License v2.0 5 votes vote down vote up 
BenchInfo parseBenchInfo(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight = parseBenchWeight(tokens);
    String name = parseBenchName(tokens);
    Benchmark bench = parseBenchClass(tokens);
    String[] args = parseBenchArgs(tokens);
    if (tokens.ttype == StreamTokenizer.TT_EOL)
        tokens.nextToken();
    return new BenchInfo(bench, name, weight, args);
}
Example 19
Source File: Token.java    From jdk8u-jdk with GNU General Public License v2.0 5 votes vote down vote up 
public String toString() {
    StringBuilder sb = new StringBuilder();
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        sb.append("ttype=TT_EOL");
        break;
    case StreamTokenizer.TT_EOF:
        sb.append("ttype=TT_EOF");
        break;
    case StreamTokenizer.TT_NUMBER:
        sb.append("ttype=TT_NUM,").append("nval="+nval);
        break;
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            sb.append("ttype=TT_WORD:IDENTIFIER");
        } else {
            sb.append("ttype=TT_WORD:").append("sval="+sval);
        }
        break;
    default:
        if (ttype == (int)'"') {
            sb.append("ttype=TT_STRING:").append("sval="+sval);
        } else {
            sb.append("ttype=TT_CHAR:").append((char)ttype);
        }
        break;
    }
    return sb.toString();
}
Example 20
Source File: ExpressionEndState.java    From reladomo with Apache License 2.0 4 votes vote down vote up 
@Override
public ComputedAttributeParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    ComputedAttributeParserState nextState = null;
    while(nextState == null && st.ttype != StreamTokenizer.TT_EOF)
    {
        int nextToken = st.nextToken();
        if (nextToken != StreamTokenizer.TT_EOL && nextToken != StreamTokenizer.TT_EOF)
        {
            ArrayList<Expression> stack = this.getParser().getStateStack();
            switch(nextToken)
            {
                case '.':
                    Expression expression = stack.remove(stack.size() - 1);
                    stack.add(new FunctionExpression(expression));
                    nextState = new FunctionBeginExpressionState(this.getParser());
                    break;
                case ',':
                    addParameterToFunction(st, stack);
                    nextState = new ExpressionBeginState(this.getParser());
                    break;
                case ';':
                    endCaseValue(st, stack);
                    nextState = new CaseSelectorBeginParserState(this.getParser());
                    break;
                case ')':
                    endFunctionOrCase(st, stack);
                    // nextState = new ExpressionEndState(this.getParser()); we can just loop here instead
                    break;
                case StreamTokenizer.TT_NUMBER:
                    throw new ParseException("unexpected number "+st.nval+" in expression "+this.getParser().getFormula()+" in "+this.getParser().getDiagnosticMessage());
                case StreamTokenizer.TT_WORD:
                    throw new ParseException("unexpected word "+st.sval+" in expression "+this.getParser().getFormula()+" in "+this.getParser().getDiagnosticMessage());
                default:
                    char ch = (char)st.ttype;
                    throw createUnexpectedCharacterException(ch, ",;).");
            }
        }
    }
    return nextState;
}