Java Code Examples for java.io.StreamTokenizer#eolIsSignificant()
The following examples show how to use
java.io.StreamTokenizer#eolIsSignificant() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ArffLoader.java From incubator-samoa with Apache License 2.0 | 6 votes |
|
/**
* Instantiates a new arff loader.
*
* @param reader the reader
* @param range
*/
public ArffLoader(Reader reader, Range range) {
this.range = range;
BufferedReader br = new BufferedReader(reader);
//Init streamTokenizer
streamTokenizer = new StreamTokenizer(br);
streamTokenizer.resetSyntax();
streamTokenizer.whitespaceChars(0, ' ');
streamTokenizer.wordChars(' ' + 1, '\u00FF');
streamTokenizer.whitespaceChars(',', ',');
streamTokenizer.commentChar('%');
streamTokenizer.quoteChar('"');
streamTokenizer.quoteChar('\'');
streamTokenizer.ordinaryChar('{');
streamTokenizer.ordinaryChar('}');
streamTokenizer.eolIsSignificant(true);
this.instanceInformation = this.getHeader();
if (range != null) { //is MultiLabel
this.instanceInformation.setRangeOutputIndices(range);
}
}
Example 2
| Source File: ArffLoader.java From samoa with Apache License 2.0 | 6 votes |
|
private void initStreamTokenizer(Reader reader) {
BufferedReader br = new BufferedReader(reader);
//Init streamTokenizer
streamTokenizer = new StreamTokenizer(br);
streamTokenizer.resetSyntax();
streamTokenizer.whitespaceChars(0, ' ');
streamTokenizer.wordChars(' ' + 1, '\u00FF');
streamTokenizer.whitespaceChars(',', ',');
streamTokenizer.commentChar('%');
streamTokenizer.quoteChar('"');
streamTokenizer.quoteChar('\'');
streamTokenizer.ordinaryChar('{');
streamTokenizer.ordinaryChar('}');
streamTokenizer.eolIsSignificant(true);
this.instanceInformation = this.getHeader();
if (classAttribute < 0) {
this.instanceInformation.setClassIndex(this.instanceInformation.numAttributes() - 1);
//System.out.print(this.instanceInformation.classIndex());
} else if (classAttribute > 0) {
this.instanceInformation.setClassIndex(classAttribute - 1);
}
}
Example 3
| Source File: CSVLoader.java From tsml with GNU General Public License v3.0 | 6 votes |
|
/**
* Initializes the stream tokenizer.
*
* @param tokenizer the tokenizer to initialize
*/
private void initTokenizer(StreamTokenizer tokenizer) {
tokenizer.resetSyntax();
tokenizer.whitespaceChars(0, (' ' - 1));
tokenizer.wordChars(' ', '\u00FF');
tokenizer.whitespaceChars(m_FieldSeparator.charAt(0),
m_FieldSeparator.charAt(0));
// tokenizer.commentChar('%');
String[] parts = m_Enclosures.split(",");
for (String e : parts) {
if (e.length() > 1 || e.length() == 0) {
throw new IllegalArgumentException(
"Enclosures can only be single characters");
}
tokenizer.quoteChar(e.charAt(0));
}
tokenizer.eolIsSignificant(true);
}
Example 4
| Source File: Algorithm.java From KEEL with GNU General Public License v3.0 | 6 votes |
|
/** Function to initialize the stream tokenizer.
*
* @param tokenizer The tokenizer.
*/
protected void initTokenizer( StreamTokenizer tokenizer )
{
tokenizer.resetSyntax();
tokenizer.whitespaceChars( 0, ' ' );
tokenizer.wordChars( ' '+1,'\u00FF' );
tokenizer.whitespaceChars( ',',',' );
tokenizer.quoteChar( '"' );
tokenizer.quoteChar( '\'' );
tokenizer.ordinaryChar( '=' );
tokenizer.ordinaryChar( '{' );
tokenizer.ordinaryChar( '}' );
tokenizer.ordinaryChar( '[' );
tokenizer.ordinaryChar( ']' );
tokenizer.eolIsSignificant( true );
}
Example 5
| Source File: ArffLoader.java From moa with GNU General Public License v3.0 | 6 votes |
|
/**
* Instantiates a new arff loader.
*
* @param reader the reader
* @param range the range
*/
public ArffLoader(Reader reader, Range range) {
this.range = range;
BufferedReader br = new BufferedReader(reader);
//Init streamTokenizer
streamTokenizer = new StreamTokenizer(br);
streamTokenizer.resetSyntax();
streamTokenizer.whitespaceChars(0, ' ');
streamTokenizer.wordChars(' ' + 1, '\u00FF');
streamTokenizer.whitespaceChars(',', ',');
streamTokenizer.commentChar('%');
streamTokenizer.quoteChar('"');
streamTokenizer.quoteChar('\'');
streamTokenizer.ordinaryChar('{');
streamTokenizer.ordinaryChar('}');
streamTokenizer.eolIsSignificant(true);
this.instanceInformation = this.getHeader();
if (range != null) { //is MultiLabel
this.instanceInformation.setRangeOutputIndices(range);
}
}
Example 6
| Source File: ReadAhead.java From jdk8u-jdk with GNU General Public License v2.0 | 5 votes |
|
private static void test(StreamTokenizer st) throws Exception {
st.eolIsSignificant(true);
int tt = st.nextToken();
if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
tt = st.nextToken();
if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
Example 7
| Source File: FileSourceBase.java From gama with GNU General Public License v3.0 | 5 votes |
|
/**
* Method to override to configure the tokenizer behaviour. It is called each time a tokenizer is created (for the
* parsed file and all included files).
*/
protected void configureTokenizer(final StreamTokenizer tok) throws IOException {
if (COMMENT_CHAR > 0) {
tok.commentChar(COMMENT_CHAR);
}
tok.quoteChar(QUOTE_CHAR);
tok.eolIsSignificant(eol_is_significant);
tok.wordChars('_', '_');
tok.parseNumbers();
}
Example 8
| Source File: ReadAhead.java From openjdk-jdk8u with GNU General Public License v2.0 | 5 votes |
|
private static void test(StreamTokenizer st) throws Exception {
st.eolIsSignificant(true);
int tt = st.nextToken();
if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
tt = st.nextToken();
if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
Example 9
| Source File: ReadAhead.java From jdk8u-jdk with GNU General Public License v2.0 | 5 votes |
|
private static void test(StreamTokenizer st) throws Exception {
st.eolIsSignificant(true);
int tt = st.nextToken();
if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
tt = st.nextToken();
if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
Example 10
| Source File: ReadAhead.java From hottub with GNU General Public License v2.0 | 5 votes |
|
private static void test(StreamTokenizer st) throws Exception {
st.eolIsSignificant(true);
int tt = st.nextToken();
if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
tt = st.nextToken();
if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
Example 11
| Source File: Json.java From netcdf-java with BSD 3-Clause "New" or "Revised" License | 5 votes |
|
public Object parse(String text) throws IOException {
StringReader rdr = new StringReader(text);
StreamTokenizer tokens = new StreamTokenizer(rdr);
tokens.eolIsSignificant(false);
tokens.quoteChar(QUOTE);
tokens.wordChars('a', 'z');
tokens.wordChars('A', 'Z');
tokens.wordChars('0', '9');
tokens.wordChars('_', '_');
tokens.wordChars('-', '-');
tokens.wordChars('$', '$');
Object result = parseR(tokens);
return result;
}
Example 12
| Source File: Board.java From megamek with GNU General Public License v2.0 | 5 votes |
|
/**
* Inspect specified board file and return its dimensions.
*
* @param filepath
* The path to the board file.
* @return A {@link BoardDimensions} object containing the dimension.
*/
public static BoardDimensions getSize(final File filepath) {
int boardx = 0;
int boardy = 0;
try {
// make inpustream for board
Reader r = new BufferedReader(new FileReader(filepath));
// read board, looking for "size"
StreamTokenizer st = new StreamTokenizer(r);
st.eolIsSignificant(true);
st.commentChar('#');
st.quoteChar('"');
st.wordChars('_', '_');
while (st.nextToken() != StreamTokenizer.TT_EOF) {
if ((st.ttype == StreamTokenizer.TT_WORD) && st.sval.equalsIgnoreCase("size")) {
st.nextToken();
boardx = (int) st.nval;
st.nextToken();
boardy = (int) st.nval;
break;
}
}
r.close();
} catch (IOException ex) {
return null;
}
return new BoardDimensions(boardx, boardy);
}
Example 13
| Source File: XYZApp.java From jdk8u_jdk with GNU General Public License v2.0 | 4 votes |
|
/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
this();
StreamTokenizer st = new StreamTokenizer(
new BufferedReader(new InputStreamReader(is, "UTF-8")));
st.eolIsSignificant(true);
st.commentChar('#');
try {
scan:
while (true) {
switch (st.nextToken()) {
case StreamTokenizer.TT_EOF:
break scan;
default:
break;
case StreamTokenizer.TT_WORD:
String name = st.sval;
double x = 0,
y = 0,
z = 0;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
x = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
y = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
z = st.nval;
}
}
}
addVert(name, (float) x, (float) y, (float) z);
while (st.ttype != StreamTokenizer.TT_EOL
&& st.ttype != StreamTokenizer.TT_EOF) {
st.nextToken();
}
} // end Switch
} // end while
is.close();
} // end Try
catch (IOException e) {
}
if (st.ttype != StreamTokenizer.TT_EOF) {
throw new Exception(st.toString());
}
}
Example 14
| Source File: XYZApp.java From hottub with GNU General Public License v2.0 | 4 votes |
|
/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
this();
StreamTokenizer st = new StreamTokenizer(
new BufferedReader(new InputStreamReader(is, "UTF-8")));
st.eolIsSignificant(true);
st.commentChar('#');
try {
scan:
while (true) {
switch (st.nextToken()) {
case StreamTokenizer.TT_EOF:
break scan;
default:
break;
case StreamTokenizer.TT_WORD:
String name = st.sval;
double x = 0,
y = 0,
z = 0;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
x = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
y = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
z = st.nval;
}
}
}
addVert(name, (float) x, (float) y, (float) z);
while (st.ttype != StreamTokenizer.TT_EOL
&& st.ttype != StreamTokenizer.TT_EOF) {
st.nextToken();
}
} // end Switch
} // end while
is.close();
} // end Try
catch (IOException e) {
}
if (st.ttype != StreamTokenizer.TT_EOF) {
throw new Exception(st.toString());
}
}
Example 15
| Source File: OldStreamTokenizerTest.java From j2objc with Apache License 2.0 | 4 votes |
|
public void test_basicStringTokenizerMethods() throws IOException {
String str = "Testing 12345 \n alpha \r\n omega";
String strb = "-3.8 'BLIND mice' \r sEe /* how */ they run";
StringReader aa = new StringReader(str);
StringReader ba = new StringReader(strb);
StreamTokenizer a = new StreamTokenizer(aa);
StreamTokenizer b = new StreamTokenizer(ba);
Assert.assertTrue(a.lineno() == 1);
Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
Assert.assertTrue(a.toString().equals("Token[Testing], line 1"));
Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_NUMBER);
Assert.assertTrue(a.toString().equals("Token[n=12345.0], line 1"));
Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
Assert.assertTrue(a.toString().equals("Token[alpha], line 2"));
Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
Assert.assertTrue(a.toString().equals("Token[omega], line 3"));
Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_EOF);
Assert.assertTrue(a.toString().equals("Token[EOF], line 3"));
b.commentChar('u');
b.eolIsSignificant(true);
b.lowerCaseMode(true);
b.ordinaryChar('y');
b.slashStarComments(true);
Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_NUMBER);
Assert.assertTrue(b.nval == -3.8);
Assert.assertTrue(b.toString().equals("Token[n=-3.8], line 1"));
Assert.assertTrue(b.nextToken() == 39); // '
Assert.assertTrue(b.toString().equals("Token[BLIND mice], line 1"));
Assert.assertTrue(b.nextToken() == 10); // \n
Assert.assertTrue(b.toString().equals("Token[EOL], line 2"));
Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
Assert.assertTrue(b.toString().equals("Token[see], line 2"));
Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
Assert.assertTrue(b.toString().equals("Token[the], line 2"));
Assert.assertTrue(b.nextToken() == 121); // y
Assert.assertTrue(b.toString().equals("Token['y'], line 2"));
Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
Assert.assertTrue(b.toString().equals("Token[r], line 2"));
Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_EOF);
Assert.assertTrue(b.toString().equals("Token[EOF], line 2"));
}
Example 16
| Source File: XYZApp.java From jdk8u-dev-jdk with GNU General Public License v2.0 | 4 votes |
|
/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
this();
StreamTokenizer st = new StreamTokenizer(
new BufferedReader(new InputStreamReader(is, "UTF-8")));
st.eolIsSignificant(true);
st.commentChar('#');
try {
scan:
while (true) {
switch (st.nextToken()) {
case StreamTokenizer.TT_EOF:
break scan;
default:
break;
case StreamTokenizer.TT_WORD:
String name = st.sval;
double x = 0,
y = 0,
z = 0;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
x = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
y = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
z = st.nval;
}
}
}
addVert(name, (float) x, (float) y, (float) z);
while (st.ttype != StreamTokenizer.TT_EOL
&& st.ttype != StreamTokenizer.TT_EOF) {
st.nextToken();
}
} // end Switch
} // end while
is.close();
} // end Try
catch (IOException e) {
}
if (st.ttype != StreamTokenizer.TT_EOF) {
throw new Exception(st.toString());
}
}
Example 17
| Source File: XYZApp.java From dragonwell8_jdk with GNU General Public License v2.0 | 4 votes |
|
/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
this();
StreamTokenizer st = new StreamTokenizer(
new BufferedReader(new InputStreamReader(is, "UTF-8")));
st.eolIsSignificant(true);
st.commentChar('#');
try {
scan:
while (true) {
switch (st.nextToken()) {
case StreamTokenizer.TT_EOF:
break scan;
default:
break;
case StreamTokenizer.TT_WORD:
String name = st.sval;
double x = 0,
y = 0,
z = 0;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
x = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
y = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
z = st.nval;
}
}
}
addVert(name, (float) x, (float) y, (float) z);
while (st.ttype != StreamTokenizer.TT_EOL
&& st.ttype != StreamTokenizer.TT_EOF) {
st.nextToken();
}
} // end Switch
} // end while
is.close();
} // end Try
catch (IOException e) {
}
if (st.ttype != StreamTokenizer.TT_EOF) {
throw new Exception(st.toString());
}
}
Example 18
| Source File: XYZApp.java From openjdk-8 with GNU General Public License v2.0 | 4 votes |
|
/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
this();
StreamTokenizer st = new StreamTokenizer(
new BufferedReader(new InputStreamReader(is, "UTF-8")));
st.eolIsSignificant(true);
st.commentChar('#');
try {
scan:
while (true) {
switch (st.nextToken()) {
case StreamTokenizer.TT_EOF:
break scan;
default:
break;
case StreamTokenizer.TT_WORD:
String name = st.sval;
double x = 0,
y = 0,
z = 0;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
x = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
y = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
z = st.nval;
}
}
}
addVert(name, (float) x, (float) y, (float) z);
while (st.ttype != StreamTokenizer.TT_EOL
&& st.ttype != StreamTokenizer.TT_EOF) {
st.nextToken();
}
} // end Switch
} // end while
is.close();
} // end Try
catch (IOException e) {
}
if (st.ttype != StreamTokenizer.TT_EOF) {
throw new Exception(st.toString());
}
}
Example 19
| Source File: CSVBuilder.java From OpenForecast with GNU Lesser General Public License v2.1 | 4 votes |
|
/**
* Builds a DataPoint from the given CSV line. This method should only be
* used to parse a line that is expected to be made up of numeric data
* only. Use {@link #readHeaderRow} to read a header row if one is expected.
* @param line the input line of comma separated values to parse and use
* to construct a new DataPoint.
* @return a DataPoint object with values as specified by the given input
* String.
*/
private DataPoint build( String line )
throws IOException
{
Observation dataPoint = new Observation( 0.0 );
StreamTokenizer tokenizer
= new StreamTokenizer( new StringReader( line ) );
tokenizer.commentChar( '#' );
tokenizer.eolIsSignificant( true );
tokenizer.parseNumbers();
int i = 0;
int n = getNumberOfVariables();
int lastToken = SEPARATOR;
do
{
// Read next token
tokenizer.nextToken();
switch ( tokenizer.ttype )
{
case '\t':
case ' ':
// Skip whitespace
continue;
case SEPARATOR:
// Check for two adjacent commas
if ( lastToken != SEPARATOR )
break;
// Two adjacent commas. Assume 0.0 between them
tokenizer.nval = 0.0;
// Fall through, and handle as a number
case StreamTokenizer.TT_NUMBER:
// Handle numbers appropriately as data
// If this is the last value on the line, treat it
// as the dependent variable value
if ( i == n )
dataPoint.setDependentValue(tokenizer.nval);
else
dataPoint.setIndependentValue(getVariableName(i),
tokenizer.nval);
i++;
break;
case StreamTokenizer.TT_WORD:
throw new IOException( "Invalid input in CSV file. Number expected, found '"+tokenizer.sval+"'");
case StreamTokenizer.TT_EOL:
case StreamTokenizer.TT_EOF:
break;
default:
}
lastToken = tokenizer.ttype;
}
while ( tokenizer.ttype != StreamTokenizer.TT_EOF );
return dataPoint;
}
Example 20
| Source File: Harness.java From TencentKona-8 with GNU General Public License v2.0 | 3 votes |
|
/**
* Create new benchmark harness with given configuration and reporter.
* Throws ConfigFormatException if there was an error parsing the config
* file.
* <p>
* <b>Config file syntax:</b>
* <p>
* '#' marks the beginning of a comment. Blank lines are ignored. All
* other lines should adhere to the following format:
* <pre>
* <weight> <name> <class> [<args>]
* </pre>
* <weight> is a floating point value which is multiplied times the
* benchmark's execution time to determine its weighted score. The
* total score of the benchmark suite is the sum of all weighted scores
* of its benchmarks.
* <p>
* <name> is a name used to identify the benchmark on the benchmark
* report. If the name contains whitespace, the quote character '"' should
* be used as a delimiter.
* <p>
* <class> is the full name (including the package) of the class
* containing the benchmark implementation. This class must implement
* bench.Benchmark.
* <p>
* [<args>] is a variable-length list of runtime arguments to pass to
* the benchmark. Arguments containing whitespace should use the quote
* character '"' as a delimiter.
* <p>
* <b>Example:</b>
* <pre>
* 3.5 "My benchmark" bench.serial.Test first second "third arg"
* </pre>
*/
public Harness(InputStream in) throws IOException, ConfigFormatException {
Vector bvec = new Vector();
StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));
tokens.resetSyntax();
tokens.wordChars(0, 255);
tokens.whitespaceChars(0, ' ');
tokens.commentChar('#');
tokens.quoteChar('"');
tokens.eolIsSignificant(true);
tokens.nextToken();
while (tokens.ttype != StreamTokenizer.TT_EOF) {
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"': // parse line
bvec.add(parseBenchInfo(tokens));
break;
default: // ignore
tokens.nextToken();
break;
}
}
binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}
