Lines Matching defs:tokenizer
314 final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(str));
315 tokenizer.resetSyntax();
316 tokenizer.wordChars(0, 255);
317 tokenizer.whitespaceChars(0, ' ');
318 tokenizer.commentChar('#');
319 tokenizer.quoteChar('"');
320 tokenizer.quoteChar('\'');
323 while (nextToken(tokenizer) != StreamTokenizer.TT_EOF) {
324 final String s = tokenizer.sval;
325 // The tokenizer understands about honoring quoted strings and recognizes
343 private static int nextToken(final StreamTokenizer tokenizer) {
345 return tokenizer.nextToken();