Java Code Examples for java.io.StreamTokenizer#nextToken()

Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 

private static List<String> getTokens(StreamTokenizer st) {
    final List<String> tokens = new LinkedList<>();
    try {
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            if (st.ttype == StreamTokenizer.TT_NUMBER) {
                tokens.add(String.valueOf(st.nval));
            } else if (st.ttype == StreamTokenizer.TT_WORD) {
                tokens.add(st.sval.trim());
            } else if (st.sval != null) {
                if (((char) st.ttype) == '"' || ((char) st.ttype) == '\'') {
                    tokens.add('"' + st.sval + '"');
                } else {
                    tokens.add(st.sval.trim());
                }
            } else if ((char) st.ttype != ' ') {
                tokens.add(Character.toString((char) st.ttype));
            }
        }
    } catch (IOException e) {
        logger.warn("Swallowing IOException", e); // TODO: Why are we swallowing this?
    }
    return tokens;
}
 

private String getUnquotedString(String str) {
  if ((str == null) || !(str.startsWith("\"") && str.endsWith("\""))) return str;

  new StreamTokenizer(new StringReader(str));
  StreamTokenizer parser = new StreamTokenizer(new StringReader(str));
  String result;

  try {
    parser.nextToken();
    if (parser.ttype == '"') {
      result = parser.sval;
    } else {
      result = "ERROR!";
    }
  } catch (IOException e) {
    result = null;

    LoggerHelper.info("Encountered an IOException while parsing the input string", e);
  }

  return result;
}
 

/**
 * Parses a option line likes
 *    -genkaypair -dname "CN=Me"
 * and add the results into a list
 * @param list the list to fill into
 * @param s the line
 */
private static void parseArgsLine(List<String> list, String s)
        throws IOException, PropertyExpander.ExpandException {
    StreamTokenizer st = new StreamTokenizer(new StringReader(s));

    st.resetSyntax();
    st.whitespaceChars(0x00, 0x20);
    st.wordChars(0x21, 0xFF);
    // Everything is a word char except for quotation and apostrophe
    st.quoteChar('"');
    st.quoteChar('\'');

    while (true) {
        if (st.nextToken() == StreamTokenizer.TT_EOF) {
            break;
        }
        list.add(PropertyExpander.expand(st.sval));
    }
}
 

String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 

String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 

/**
 * Read an int from the command file
 * Negative numbers are preceded by "-"
 */
private int getInt(StreamTokenizer tkn) throws IOException
{
    int mult = 1;
    int val = tkn.nextToken();
    if (tkn.sval != null && tkn.sval.equals("-"))
    {
        mult = -1;
        val = tkn.nextToken();
    }

    if (val != StreamTokenizer.TT_NUMBER)
    {
        assertNotNull("Invalid string on line " + ln(tkn), tkn.sval);
        String str = tkn.sval.toLowerCase(Locale.ENGLISH);
        if (!str.startsWith("0x")) {
            fail("Expecting number, got " + tkn.sval + " on line " +
                    ln(tkn));
        } else {
            return convertHex(str, ln(tkn));
        }
    }
    return (new Double(tkn.nval).intValue() * mult);
}
 

private void parseAttributes(StreamTokenizer st) throws IOException, ParseException
{
    int token = st.nextToken();
    boolean wantAttribute = true;
    while(token != StreamTokenizer.TT_EOL)
    {
        if (wantAttribute)
        {
            if (token != StreamTokenizer.TT_WORD)
            {
                throw new ParseException("expected an attribute name on line "+st.lineno(), st.lineno());
            }
            this.getParser().getCurrentParsedData().addAttribute(st.sval, st.lineno());
        }
        else
        {
            if (token != ',')
            {
                throw new ParseException("Expected a comma on line "+st.lineno(), st.lineno());
            }
        }
        wantAttribute = !wantAttribute;
        token = st.nextToken();
    }
    if (wantAttribute)
    {
        throw new ParseException("extra comma at the end of line "+st.lineno(), st.lineno());
    }
}
 

/** Delivers the next token and skip empty lines. */
public static void getFirstToken(StreamTokenizer tokenizer) throws IOException {

	while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
		// skip empty lines
	}

	if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
		tokenizer.ttype = StreamTokenizer.TT_WORD;
	} else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
		tokenizer.ttype = '?';
	}
}
 

/** Skips all tokens before next end of line (EOL). */
public static void waitForEOL(StreamTokenizer tokenizer) throws IOException {

	while (tokenizer.nextToken() != StreamTokenizer.TT_EOL) {
		// skip everything until EOL
	}
	tokenizer.pushBack();
}
 

static Expression element(StreamTokenizer st) throws SyntaxError {
    Expression result = null;

    try {
        switch (st.nextToken()) {
            case StreamTokenizer.TT_NUMBER :
                result = new ConstantExpression(st.nval);
                break;

            case StreamTokenizer.TT_WORD :
                result = new VariableExpression(st.sval);
                break;
            case '(' :
                result = expression(st);
                st.nextToken();
                if (st.ttype != ')') {
                    st.pushBack();
                    throw new SyntaxError("Mismatched parenthesis.");
                }
                break;
            default:
                st.pushBack();
                throw new SyntaxError("Unexpected symbol on input.");
        }
    } catch (IOException ioe) {
        throw new SyntaxError("Caught an I/O exception.");
    }
    return result;
}
 

/** Delivers the next token and checks for an unexpected end of line or file. */
public static void getNextToken(StreamTokenizer tokenizer) throws IOException {
	if (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
		throw new IOException("unexpected end of line " + tokenizer.lineno());
	}

	if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
		throw new IOException("unexpected end of file in line " + tokenizer.lineno());
	} else if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
		tokenizer.ttype = StreamTokenizer.TT_WORD;
	} else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
		tokenizer.ttype = '?';
	}
}
 

BenchInfo parseBenchInfo(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight = parseBenchWeight(tokens);
    String name = parseBenchName(tokens);
    Benchmark bench = parseBenchClass(tokens);
    String[] args = parseBenchArgs(tokens);
    if (tokens.ttype == StreamTokenizer.TT_EOL)
        tokens.nextToken();
    return new BenchInfo(bench, name, weight, args);
}
 

BenchInfo parseBenchInfo(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight = parseBenchWeight(tokens);
    String name = parseBenchName(tokens);
    Benchmark bench = parseBenchClass(tokens);
    String[] args = parseBenchArgs(tokens);
    if (tokens.ttype == StreamTokenizer.TT_EOL)
        tokens.nextToken();
    return new BenchInfo(bench, name, weight, args);
}
 

@Override
public ComputedAttributeParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    ComputedAttributeParserState nextState = null;
    while(nextState == null && st.ttype != StreamTokenizer.TT_EOF)
    {
        int nextToken = st.nextToken();
        if (nextToken != StreamTokenizer.TT_EOL && nextToken != StreamTokenizer.TT_EOF)
        {
            ArrayList<Expression> stack = this.getParser().getStateStack();
            CaseExpression caseExpression = (CaseExpression) stack.get(stack.size() - 1);
            switch(nextToken)
            {
                case StreamTokenizer.TT_NUMBER:
                    caseExpression.addNumberKey(st.nval);
                    nextState = new CaseSelectorMiddleParserState(this.getParser());
                    break;
                case StreamTokenizer.TT_WORD:
                    if (st.sval.equals("null"))
                    {
                        caseExpression.addNullKey();
                        stack.add(new NullExpression());
                    }
                    else if (st.sval.equals("true") || st.sval.equals("false"))
                    {
                        caseExpression.addBooleanKey(Boolean.valueOf(st.sval));
                        stack.add(new NullExpression());
                    }
                    else if (st.sval.equals("default"))
                    {
                        caseExpression.addDefaultKey();
                        stack.add(new NullExpression());
                    }
                    else
                    {
                        throw new ParseException("unexpected word "+st.sval+" in expression "+this.getParser().getFormula()+" in "+this.getParser().getDiagnosticMessage());
                    }
                    nextState = new CaseSelectorMiddleParserState(this.getParser());
                    break;
                case '"':
                    caseExpression.addStringConstantKey(st.sval);
                    nextState = new CaseSelectorMiddleParserState(this.getParser());
                    break;
                default:
                    char ch = (char)st.ttype;
                    throw createUnexpectedCharacterException(ch, "<number>|<null>|<boolean>|<default>|<string>");
            }
        }
    }
    return nextState;
}
 

/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
    this();
    StreamTokenizer st = new StreamTokenizer(
            new BufferedReader(new InputStreamReader(is, "UTF-8")));
    st.eolIsSignificant(true);
    st.commentChar('#');

    try {
        scan:
        while (true) {
            switch (st.nextToken()) {
                case StreamTokenizer.TT_EOF:
                    break scan;
                default:
                    break;
                case StreamTokenizer.TT_WORD:
                    String name = st.sval;
                    double x = 0,
                     y = 0,
                     z = 0;
                    if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                        x = st.nval;
                        if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                            y = st.nval;
                            if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                                z = st.nval;
                            }
                        }
                    }
                    addVert(name, (float) x, (float) y, (float) z);
                    while (st.ttype != StreamTokenizer.TT_EOL
                            && st.ttype != StreamTokenizer.TT_EOF) {
                        st.nextToken();
                    }

            }   // end Switch

        }  // end while

        is.close();

    } // end Try
    catch (IOException e) {
    }

    if (st.ttype != StreamTokenizer.TT_EOF) {
        throw new Exception(st.toString());
    }

}
 

/** Function to read the options from the execution file and assign
 * the values to the parameters.
 *
 * @param options 		The StreamTokenizer that reads the parameters file.
 *
 * @throws Exception	If the format of the file is not correct.
 */
protected void setOptions(StreamTokenizer options) throws Exception {
	options.nextToken();

	/* Checks that the file starts with the token algorithm */
	if (options.sval.equalsIgnoreCase("algorithm")) {
		options.nextToken();
		options.nextToken();

		/* Check algorithm name
		if (!options.sval.equalsIgnoreCase("CART")) {
			throw new Exception("The name of the algorithm is not correct.");
		}
		 */

		options.nextToken();
		options.nextToken();

		/* Reads the names of the input files*/
		if (options.sval.equalsIgnoreCase("inputData")) {
			options.nextToken();
			options.nextToken();
			modelFileName = options.sval;

			if (options.nextToken() != StreamTokenizer.TT_EOL) {
				trainFileName = options.sval;
				options.nextToken();
				testFileName = options.sval;
				if (options.nextToken() != StreamTokenizer.TT_EOL) {
					trainFileName = modelFileName;
					options.nextToken();
				}
			}
		} else {
			throw new Exception("No file test provided.");
		}

		/* Reads the names of the output files*/
		while (true) {
			if (options.nextToken() == StreamTokenizer.TT_EOF) {
				throw new Exception("No output file provided.");
			}

			if (options.sval == null) {
				continue;
			} else if (options.sval.equalsIgnoreCase("outputData")) {
				break;
			}
		}

		options.nextToken();
		options.nextToken();
		trainOutputFileName = options.sval;
		options.nextToken();
		testOutputFileName = options.sval;
		options.nextToken();
		resultFileName = options.sval;

		if (!getNextToken(options)) {
			throw new Exception("No instances provided.");
		}

		if (options.ttype == StreamTokenizer.TT_EOF) {
			return;
		}

		for (int k = 0; k < nParam; k++) {

			/* Reads the maxDepth parameter */
			if (options.sval.equalsIgnoreCase("maxDepth")) {
				options.nextToken();
				options.nextToken();

				if (Integer.parseInt(options.sval) > 0) {
					maxDepth = Integer.parseInt(options.sval);
				}

				if (!getNextToken(options)) {
					return;
				} else {
					continue;
				}
			}
			
			/* Any other parameter should be added here */

		} // end for
	} else {
		throw new Exception("The file must start with the word " +
		"algorithm followed of the name of the algorithm.");
	}
}
 

/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
    this();
    StreamTokenizer st = new StreamTokenizer(
            new BufferedReader(new InputStreamReader(is, "UTF-8")));
    st.eolIsSignificant(true);
    st.commentChar('#');

    try {
        scan:
        while (true) {
            switch (st.nextToken()) {
                case StreamTokenizer.TT_EOF:
                    break scan;
                default:
                    break;
                case StreamTokenizer.TT_WORD:
                    String name = st.sval;
                    double x = 0,
                     y = 0,
                     z = 0;
                    if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                        x = st.nval;
                        if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                            y = st.nval;
                            if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                                z = st.nval;
                            }
                        }
                    }
                    addVert(name, (float) x, (float) y, (float) z);
                    while (st.ttype != StreamTokenizer.TT_EOL
                            && st.ttype != StreamTokenizer.TT_EOF) {
                        st.nextToken();
                    }

            }   // end Switch

        }  // end while

        is.close();

    } // end Try
    catch (IOException e) {
    }

    if (st.ttype != StreamTokenizer.TT_EOF) {
        throw new Exception(st.toString());
    }

}
 

void parseCommand(StreamTokenizer in) throws IOException {
    if (in.nextToken() != StreamTokenizer.TT_WORD) {
        throw new IOException("Unexpected token: " + in.ttype);
    }
    String word;
    String[] aliasBuf = null;
    int aliasPos;
    if (aliases != null
            && (aliasBuf = (String[]) aliases.get(in.sval)) != null) {
        word = aliasBuf[0];
        aliasPos = 1;
    } else {
        word = in.sval;
        aliasPos = 0;
    }
    if ("".equals(groupName) || word.startsWith("/")) {
        if (word.startsWith("/"))
            word = word.substring(1);
        commandGroupRef = matchCommandGroup(bc, word);
        word = null;
    } else {
        if (SessionCommandGroup.NAME.equals(groupName)) {
            commandGroupRef = null;
        } else {
            Collection<ServiceReference<CommandGroup>> refs = null;
            try {
                refs = bc.getServiceReferences(CommandGroup.class,
                                               "(groupName=" + groupName + ")");
            } catch (InvalidSyntaxException ignore) {
            }
            if (refs.isEmpty()) {
                throw new IOException("No such command group: " + groupName);
            }
            commandGroupRef = refs.iterator().next();
        }
    }
    ArrayList<String> vargs = new ArrayList<String>();
    boolean done = false;
    while (!done) {
        if (word != null) {
            vargs.add(word);
            word = null;
        } else if (aliasPos > 0) {
            if (aliasPos < aliasBuf.length) {
                word = aliasBuf[aliasPos++];
            } else {
                aliasPos = 0;
            }
        } else {
            switch (in.nextToken()) {
            case '|':
                out = new Pipe();
                isPiped = true;
                done = true;
                break;
            case '&':
                isBackground = true;
            case ';':
                done = true;
                break;
            case '<':
                if (in.nextToken() != StreamTokenizer.TT_WORD) {
                    throw new IOException("Empty input redirect");
                }
                // Set input to file
                break;
            case '>':
                if (in.nextToken() != StreamTokenizer.TT_WORD) {
                    throw new IOException("Empty output redirect");
                }
                // Set output to file
                break;
            case StreamTokenizer.TT_EOL:
            case StreamTokenizer.TT_EOF:
                in.pushBack();
                done = true;
                break;
            case '"':
            case '\'':
            case StreamTokenizer.TT_WORD:
                word = in.sval;
                break;
            default:
                throw new IOException("Unknown token: " + in.ttype);
            }
        }
    }
    args = vargs.toArray(new String[vargs.size()]);
}
 

/**
 * Create new benchmark harness with given configuration and reporter.
 * Throws ConfigFormatException if there was an error parsing the config
 * file.
 * <p>
 * <b>Config file syntax:</b>
 * <p>
 * '#' marks the beginning of a comment.  Blank lines are ignored.  All
 * other lines should adhere to the following format:
 * <pre>
 *     &lt;weight&gt; &lt;name&gt; &lt;class&gt; [&lt;args&gt;]
 * </pre>
 * &lt;weight&gt; is a floating point value which is multiplied times the
 * benchmark's execution time to determine its weighted score.  The
 * total score of the benchmark suite is the sum of all weighted scores
 * of its benchmarks.
 * <p>
 * &lt;name&gt; is a name used to identify the benchmark on the benchmark
 * report.  If the name contains whitespace, the quote character '"' should
 * be used as a delimiter.
 * <p>
 * &lt;class&gt; is the full name (including the package) of the class
 * containing the benchmark implementation.  This class must implement
 * bench.Benchmark.
 * <p>
 * [&lt;args&gt;] is a variable-length list of runtime arguments to pass to
 * the benchmark.  Arguments containing whitespace should use the quote
 * character '"' as a delimiter.
 * <p>
 * <b>Example:</b>
 * <pre>
 *      3.5 "My benchmark" bench.serial.Test first second "third arg"
 * </pre>
 */
public Harness(InputStream in) throws IOException, ConfigFormatException {
    Vector bvec = new Vector();
    StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));

    tokens.resetSyntax();
    tokens.wordChars(0, 255);
    tokens.whitespaceChars(0, ' ');
    tokens.commentChar('#');
    tokens.quoteChar('"');
    tokens.eolIsSignificant(true);

    tokens.nextToken();
    while (tokens.ttype != StreamTokenizer.TT_EOF) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_WORD:
            case '"':                       // parse line
                bvec.add(parseBenchInfo(tokens));
                break;

            default:                        // ignore
                tokens.nextToken();
                break;
        }
    }
    binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}