94341fa76202bc718b751b19701497e1863f6cf5
1 /* StreamTokenizer.java -- parses streams of characters into tokens
2 Copyright (C) 1998, 1999, 2000, 2001, 2002 Free Software Foundation
4 This file is part of GNU Classpath.
6 GNU Classpath is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2, or (at your option)
11 GNU Classpath is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GNU Classpath; see the file COPYING. If not, write to the
18 Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
21 Linking this library statically or dynamically with other modules is
22 making a combined work based on this library. Thus, the terms and
23 conditions of the GNU General Public License cover the whole
26 As a special exception, the copyright holders of this library give you
27 permission to link this library with independent modules to produce an
28 executable, regardless of the license terms of these independent
29 modules, and to copy and distribute the resulting executable under
30 terms of your choice, provided that you also meet, for each linked
31 independent module, the terms and conditions of the license of that
32 module. An independent module is a module which is not derived from
33 or based on this library. If you modify this library, you may extend
34 this exception to your version of the library, but you are not
35 obligated to do so. If you do not wish to do so, delete this
36 exception statement from your version. */
41 * This class parses streams of characters into tokens. There are a
42 * million-zillion flags that can be set to control the parsing, as
43 * described under the various method headings.
45 * @author Warren Levy <warrenl@cygnus.com>
46 * @date October 25, 1998.
48 /* Written using "Java Class Libraries", 2nd edition, ISBN 0-201-31002-3
49 * "The Java Language Specification", ISBN 0-201-63451-1
50 * plus online API docs for JDK 1.2 beta from http://www.javasoft.com.
51 * Status: Believed complete and correct.
54 public class StreamTokenizer
56 /** A constant indicating that the end of the stream has been read. */
57 public static final int TT_EOF
= -1;
59 /** A constant indicating that the end of the line has been read. */
60 public static final int TT_EOL
= '\n';
62 /** A constant indicating that a number token has been read. */
63 public static final int TT_NUMBER
= -2;
65 /** A constant indicating that a word token has been read. */
66 public static final int TT_WORD
= -3;
68 /** A constant indicating that no tokens have been read yet. */
69 private static final int TT_NONE
= -4;
72 * Contains the type of the token read resulting from a call to nextToken
73 * The rules are as follows:
75 * <li>For a token consisting of a single ordinary character, this is the
76 * value of that character.
77 * <li>For a quoted string, this is the value of the quote character
78 * <li>For a word, this is TT_WORD
79 * <li>For a number, this is TT_NUMBER
80 * <li>For the end of the line, this is TT_EOL
81 * <li>For the end of the stream, this is TT_EOF
84 public int ttype
= TT_NONE
;
86 /** The String associated with word and string tokens. */
89 /** The numeric value associated with number tokens. */
92 /* Indicates whether end-of-line is recognized as a token. */
93 private boolean eolSignificant
= false;
95 /* Indicates whether word tokens are automatically made lower case. */
96 private boolean lowerCase
= false;
98 /* Indicates whether C++ style comments are recognized and skipped. */
99 private boolean slashSlash
= false;
101 /* Indicates whether C style comments are recognized and skipped. */
102 private boolean slashStar
= false;
104 /* Attribute tables of each byte from 0x00 to 0xFF. */
105 private boolean[] whitespace
= new boolean[256];
106 private boolean[] alphabetic
= new boolean[256];
107 private boolean[] numeric
= new boolean[256];
108 private boolean[] quote
= new boolean[256];
109 private boolean[] comment
= new boolean[256];
111 /* The Reader associated with this class. */
112 private PushbackReader in
;
114 /* Indicates if a token has been pushed back. */
115 private boolean pushedBack
= false;
117 /* Contains the current line number of the reader. */
118 private int lineNumber
= 1;
121 * This method reads bytes from an <code>InputStream</code> and tokenizes
122 * them. For details on how this method operates by default, see
123 * <code>StreamTokenizer(Reader)</code>.
125 * @param in The <code>InputStream</code> to read from
127 * @deprecated Since JDK 1.1.
129 public StreamTokenizer(InputStream is
)
131 this(new InputStreamReader(is
));
135 * This method initializes a new <code>StreamTokenizer</code> to read
136 * characters from a <code>Reader</code> and parse them. The char values
137 * have their hight bits masked so that the value is treated a character
138 * in the range of 0x0000 to 0x00FF.
140 * This constructor sets up the parsing table to parse the stream in the
143 * <li>The values 'A' through 'Z', 'a' through 'z' and 0xA0 through 0xFF
144 * are initialized as alphabetic
145 * <li>The values 0x00 through 0x20 are initialized as whitespace
146 * <li>The values '\'' and '"' are initialized as quote characters
147 * <li>'/' is a comment character
148 * <li>Numbers will be parsed
149 * <li>EOL is not treated as significant
150 * <li>C and C++ (//) comments are not recognized
153 * @param in The <code>Reader</code> to read chars from
155 public StreamTokenizer(Reader r
)
157 in
= new PushbackReader(r
);
159 whitespaceChars(0x00, 0x20);
162 wordChars(0xA0, 0xFF);
170 * This method sets the comment attribute on the specified character.
172 * @param c The character to set the comment attribute for, passed as an int
174 public void commentChar(int ch
)
176 if (ch
>= 0 && ch
<= 255)
181 * This method sets a flag that indicates whether or not the end of line
182 * sequence terminates and is a token. The defaults to <code>false</code>
184 * @param flag <code>true</code> if EOF is significant, <code>false</code>
187 public void eolIsSignificant(boolean flag
)
189 eolSignificant
= flag
;
193 * This method returns the current line number. Note that if the
194 * <code>pushBack()</code> method is called, it has no effect on the
195 * line number returned by this method.
197 * @return The current line number
205 * This method sets a flag that indicates whether or not alphabetic
206 * tokens that are returned should be converted to lower case.
208 * @param flag <code>true</code> to convert to lower case,
209 * <code>false</code> otherwise
211 public void lowerCaseMode(boolean flag
)
216 private boolean isWhitespace(int ch
)
218 return (ch
>= 0 && ch
<= 255 && whitespace
[ch
]);
221 private boolean isAlphabetic(int ch
)
223 return ((ch
> 255) || (ch
>= 0 && alphabetic
[ch
]));
226 private boolean isNumeric(int ch
)
228 return (ch
>= 0 && ch
<= 255 && numeric
[ch
]);
231 private boolean isQuote(int ch
)
233 return (ch
>= 0 && ch
<= 255 && quote
[ch
]);
236 private boolean isComment(int ch
)
238 return (ch
>= 0 && ch
<= 255 && comment
[ch
]);
242 * This method reads the next token from the stream. It sets the
243 * <code>ttype</code> variable to the appropriate token type and
244 * returns it. It also can set <code>sval</code> or <code>nval</code>
245 * as described below. The parsing strategy is as follows:
247 * <li>Skip any whitespace characters.
248 * <li>If a numeric character is encountered, attempt to parse a numeric
249 * value. Leading '-' characters indicate a numeric only if followed by
250 * another non-'-' numeric. The value of the numeric token is terminated
251 * by either the first non-numeric encountered, or the second occurrence of
252 * '-' or '.'. The token type returned is TT_NUMBER and <code>nval</code>
253 * is set to the value parsed.
254 * <li>If an alphabetic character is parsed, all subsequent characters
255 * are read until the first non-alphabetic or non-numeric character is
256 * encountered. The token type returned is TT_WORD and the value parsed
257 * is stored in <code>sval</code>. If lower case mode is set, the token
258 * stored in <code>sval</code> is converted to lower case. The end of line
259 * sequence terminates a word only if EOL signficance has been turned on.
260 * The start of a comment also terminates a word. Any character with a
261 * non-alphabetic and non-numeric attribute (such as white space, a quote,
262 * or a commet) are treated as non-alphabetic and terminate the word.
263 * <li>If a comment character is parsed, then all remaining characters on
264 * the current line are skipped and another token is parsed. Any EOL or
265 * EOF's encountered are not discarded, but rather terminate the comment.
266 * <li>If a quote character is parsed, then all characters up to the
267 * second occurrence of the same quote character are parsed into a
268 * <code>String</code>. This <code>String</code> is stored as
269 * <code>sval</code>, but is not converted to lower case, even if lower case
270 * mode is enabled. The token type returned is the value of the quote
271 * character encountered. Any escape sequences
272 * (\b (backspace), \t (HTAB), \n (linefeed), \f (form feed), \r
273 * (carriage return), \" (double quote), \' (single quote), \\
274 * (backslash), \XXX (octal esacpe)) are converted to the appropriate
275 * char values. Invalid esacape sequences are left in untranslated.
276 * Unicode characters like ('\ u0000') are not recognized.
277 * <li>If the C++ comment sequence "//" is encountered, and the parser
278 * is configured to handle that sequence, then the remainder of the line
279 * is skipped and another token is read exactly as if a character with
280 * the comment attribute was encountered.
281 * <li>If the C comment sequence "/*" is encountered, and the parser
282 * is configured to handle that sequence, then all characters up to and
283 * including the comment terminator sequence are discarded and another
285 * <li>If all cases above are not met, then the character is an ordinary
286 * character that is parsed as a token by itself. The char encountered
287 * is returned as the token type.
290 * @return The token type
291 * @exception IOException If an I/O error occurs
293 public int nextToken() throws IOException
298 if (ttype
!= TT_NONE
)
305 // Skip whitespace. Deal with EOL along the way.
306 while (isWhitespace(ch
= in
.read()))
307 if (ch
== '\n' || ch
== '\r')
311 // Throw away \n if in combination with \r.
312 if (ch
== '\r' && (ch
= in
.read()) != '\n')
318 return (ttype
= TT_EOL
);
322 if ((ch
= in
.read()) == '/' && slashSlash
)
324 while ((ch
= in
.read()) != '\n' && ch
!= '\r' && ch
!= TT_EOF
)
328 return nextToken(); // Recursive, but not too deep in normal cases
330 else if (ch
== '*' && slashStar
)
337 if ((ch
= in
.read()) == '/')
339 else if (ch
!= TT_EOF
)
342 else if (ch
== '\n' || ch
== '\r')
345 if (ch
== '\r' && (ch
= in
.read()) != '\n')
351 else if (ch
== TT_EOF
)
356 return nextToken(); // Recursive, but not too deep in normal cases
367 else if (isNumeric(ch
))
369 boolean isNegative
= false;
372 // Read ahead to see if this is an ordinary '-' rather than numeric.
374 if (isNumeric(ch
) && ch
!= '-')
382 return (ttype
= '-');
386 StringBuffer tokbuf
= new StringBuffer();
387 tokbuf
.append((char) ch
);
390 while (isNumeric(ch
= in
.read()) && ch
!= '-')
391 if (ch
== '.' && decCount
++ > 0)
394 tokbuf
.append((char) ch
);
401 nval
= Double
.valueOf(tokbuf
.toString()).doubleValue();
403 catch (NumberFormatException _
)
410 else if (isAlphabetic(ch
))
412 StringBuffer tokbuf
= new StringBuffer();
413 tokbuf
.append((char) ch
);
414 while (isAlphabetic(ch
= in
.read()) || isNumeric(ch
))
415 tokbuf
.append((char) ch
);
419 sval
= tokbuf
.toString();
421 sval
= sval
.toLowerCase();
423 else if (isComment(ch
))
425 while ((ch
= in
.read()) != '\n' && ch
!= '\r' && ch
!= TT_EOF
)
429 return nextToken(); // Recursive, but not too deep in normal cases.
431 else if (isQuote(ch
))
434 StringBuffer tokbuf
= new StringBuffer();
435 while ((ch
= in
.read()) != ttype
&& ch
!= '\n' && ch
!= '\r' &&
439 switch (ch
= in
.read())
455 case '\n': ch
= '\n';
457 case '\r': ch
= '\r';
465 if ((nextch
= ch1
= ch
) >= '0' && ch
<= '7')
468 if ((nextch
= in
.read()) >= '0' && nextch
<= '7')
470 ch
= ch
* 8 + nextch
- '0';
471 if ((nextch
= in
.read()) >= '0' && nextch
<= '7' &&
472 ch1
>= '0' && ch1
<= '3')
474 ch
= ch
* 8 + nextch
- '0';
480 if (nextch
!= TT_EOF
)
484 tokbuf
.append((char) ch
);
487 // Throw away matching quote char.
488 if (ch
!= ttype
&& ch
!= TT_EOF
)
491 sval
= tokbuf
.toString();
501 private void resetChar(int ch
)
503 whitespace
[ch
] = alphabetic
[ch
] = numeric
[ch
] = quote
[ch
] = comment
[ch
] =
508 * This method makes the specified character an ordinary character. This
509 * means that none of the attributes (whitespace, alphabetic, numeric,
510 * quote, or comment) will be set on this character. This character will
511 * parse as its own token.
513 * @param c The character to make ordinary, passed as an int
515 public void ordinaryChar(int ch
)
517 if (ch
>= 0 && ch
<= 255)
522 * This method makes all the characters in the specified range, range
523 * terminators included, ordinary. This means the none of the attributes
524 * (whitespace, alphabetic, numeric, quote, or comment) will be set on
525 * any of the characters in the range. This makes each character in this
526 * range parse as its own token.
528 * @param low The low end of the range of values to set the whitespace
530 * @param high The high end of the range of values to set the whitespace
533 public void ordinaryChars(int low
, int hi
)
539 for (int i
= low
; i
<= hi
; i
++)
544 * This method sets the numeric attribute on the characters '0' - '9' and
545 * the characters '.' and '-'.
547 public void parseNumbers()
549 for (int i
= 0; i
<= 9; i
++)
550 numeric
['0' + i
] = true;
557 * Puts the current token back into the StreamTokenizer so
558 * <code>nextToken</code> will return the same value on the next call.
559 * May cause the lineno method to return an incorrect value
560 * if lineno is called before the next call to nextToken.
562 public void pushBack()
568 * This method sets the quote attribute on the specified character.
570 * @param c The character to set the quote attribute for, passed as an int.
572 public void quoteChar(int ch
)
574 if (ch
>= 0 && ch
<= 255)
579 * This method removes all attributes (whitespace, alphabetic, numeric,
580 * quote, and comment) from all characters. It is equivalent to calling
581 * <code>ordinaryChars(0x00, 0xFF)</code>.
583 * @see #ordinaryChars(int, int)
585 public void resetSyntax()
587 ordinaryChars(0x00, 0xFF);
591 * This method sets a flag that indicates whether or not "C++" language style
592 * comments ("//" comments through EOL ) are handled by the parser.
593 * If this is <code>true</code> commented out sequences are skipped and
594 * ignored by the parser. This defaults to <code>false</code>.
596 * @param flag <code>true</code> to recognized and handle "C++" style
597 * comments, <code>false</code> otherwise
599 public void slashSlashComments(boolean flag
)
605 * This method sets a flag that indicates whether or not "C" language style
606 * comments (with nesting not allowed) are handled by the parser.
607 * If this is <code>true</code> commented out sequences are skipped and
608 * ignored by the parser. This defaults to <code>false</code>.
610 * @param flag <code>true</code> to recognized and handle "C" style comments,
611 * <code>false</code> otherwise
613 public void slashStarComments(boolean flag
)
619 * This method returns the current token value as a <code>String</code> in
620 * the form "Token[x], line n", where 'n' is the current line numbers and
621 * 'x' is determined as follows.
624 * <li>If no token has been read, then 'x' is "NOTHING" and 'n' is 0
625 * <li>If <code>ttype</code> is TT_EOF, then 'x' is "EOF"
626 * <li>If <code>ttype</code> is TT_EOL, then 'x' is "EOL"
627 * <li>If <code>ttype</code> is TT_WORD, then 'x' is <code>sval</code>
628 * <li>If <code>ttype</code> is TT_NUMBER, then 'x' is "n=strnval" where
629 * 'strnval' is <code>String.valueOf(nval)</code>.
630 * <li>If <code>ttype</code> is a quote character, then 'x' is
632 * <li>For all other cases, 'x' is <code>ttype</code>
635 public String
toString()
640 else if (ttype
== TT_EOL
)
642 else if (ttype
== TT_WORD
)
644 else if (ttype
== TT_NUMBER
)
645 tempstr
= "n=" + nval
;
646 else if (ttype
== TT_NONE
)
648 else // must be an ordinary char.
649 tempstr
= "\'" + (char) ttype
+ "\'";
651 return "Token[" + tempstr
+ "], line " + lineno();
655 * This method sets the whitespace attribute for all characters in the
656 * specified range, range terminators included.
658 * @param low The low end of the range of values to set the whitespace
660 * @param high The high end of the range of values to set the whitespace
663 public void whitespaceChars(int low
, int hi
)
669 for (int i
= low
; i
<= hi
; i
++)
672 whitespace
[i
] = true;
677 * This method sets the alphabetic attribute for all characters in the
678 * specified range, range terminators included.
680 * @param low The low end of the range of values to set the alphabetic
682 * @param high The high end of the range of values to set the alphabetic
685 public void wordChars(int low
, int hi
)
691 for (int i
= low
; i
<= hi
; i
++)
692 alphabetic
[i
] = true;