libjava/java/io/StreamTokenizer.java - rust-lang/gcc - Git at Google

 /* Copyright (C) 1998, 1999  Cygnus Solutions

    This file is part of libgcj.

 This software is copyrighted work licensed under the terms of the
 Libgcj License.  Please consult the file "LIBGCJ_LICENSE" for
 details.  */

 package java.io;

 /**
  * @author Warren Levy <warrenl@cygnus.com>
  * @date October 25, 1998.
  */
 /* Written using "Java Class Libraries", 2nd edition, ISBN 0-201-31002-3
  * "The Java Language Specification", ISBN 0-201-63451-1
  * plus online API docs for JDK 1.2 beta from http://www.javasoft.com.
  * Status:  Believed complete and correct.
  */

 public class StreamTokenizer
 {
   /* A constant indicating that the end of the stream has been read. */
   public static final int TT_EOF = -1;

   /* A constant indicating that the end of the line has been read. */
   public static final int TT_EOL = '\n';

   /* A constant indicating that a number token has been read. */
   public static final int TT_NUMBER = -2;

   /* A constant indicating that a word token has been read. */
   public static final int TT_WORD = -3;

   /* Contains the type of the token read resulting from a call to nextToken. */
   public int ttype;

   /* The String associated with word and string tokens. */
   public String sval;

   /* The numeric value associated with number tokens. */
   public double nval;

   /* Indicates whether end-of-line is recognized as a token. */
   private boolean eolSignificant = false;

   /* Indicates whether word tokens are automatically made lower case. */
   private boolean lowerCase = false;

   /* Indicates whether C++ style comments are recognized and skipped. */
   private boolean slashSlash = false;

   /* Indicates whether C style comments are recognized and skipped. */
   private boolean slashStar = false;

   /* Attribute tables of each byte from 0x00 to 0xFF. */
   private boolean[] whitespace;
   private boolean[] alphabetic;
   private boolean[] numeric;
   private boolean[] quote;
   private boolean[] comment;

   /* The Reader associated with this class. */
   private PushbackReader in;

   /* Indicates if a token has been pushed back. */
   private boolean pushedBack = false;

   /* Contains the current line number of the reader. */
   private int lineNumber = 1;

   // Deprecated in JDK 1.1.
   public StreamTokenizer(InputStream is)
   {
     this(new InputStreamReader(is));
   }

   public StreamTokenizer(Reader r)
   {
     in = new PushbackReader(r);

     whitespace = new boolean[256];
     alphabetic = new boolean[256];
     numeric = new boolean[256];
     quote = new boolean[256];
     comment = new boolean[256];
     for (int i = 0; i < 256; i++)
       resetChar(i);

     whitespaceChars(0x00, 0x20);
     wordChars('A', 'Z');
     wordChars('a', 'z');
     wordChars(0xA0, 0xFF);
     commentChar('/');
     quoteChar('\'');
     quoteChar('"');
     parseNumbers();
   }

   public void commentChar(int ch)
   {
     if (ch >= 0 && ch <= 255)
       comment[ch] = true;
   }

   public void eolIsSignificant(boolean flag)
   {
     eolSignificant = flag;
   }

   public int lineno()
   {
     return lineNumber;
   }

   public void lowerCaseMode(boolean flag)
   {
     lowerCase = flag;
   }

   private boolean isWhitespace(int ch)
   {
     if (ch >= 0 && ch <= 255)
       return whitespace[ch];

     return false;
   }

   private boolean isAlphabetic(int ch)
   {
     if (ch >= 0 && ch <= 255)
       return alphabetic[ch];
     else if (ch > 255)
       return true;

     return false;
   }

   private boolean isNumeric(int ch)
   {
     if (ch >= 0 && ch <= 255)
       return numeric[ch];

     return false;
   }

   private boolean isQuote(int ch)
   {
     if (ch >= 0 && ch <= 255)
       return quote[ch];

     return false;
   }

   private boolean isComment(int ch)
   {
     if (ch >= 0 && ch <= 255)
       return comment[ch];

     return false;
   }

   public int nextToken() throws IOException
   {
     if (pushedBack)
       {
 	pushedBack = false;
 	return ttype;
       }

     sval = null;
     int ch;

     // Skip whitespace.  Deal with EOL along the way.
     while (isWhitespace(ch = in.read()))
       if (ch == '\n' || ch == '\r')
 	{
 	  lineNumber++;

 	  // Throw away \n if in combination with \r.
 	  if (ch == '\r' && (ch = in.read()) != '\n')
 	    in.unread(ch);
 	  if (eolSignificant)
 	    return (ttype = TT_EOL);
 	}

     if (ch == TT_EOF)
       ttype = TT_EOF;
     else if (isNumeric(ch))
       {
 	if (ch == '-')
 	  {
 	    // Read ahead to see if this is an ordinary '-' rather than numeric.
 	    ch = in.read();
 	    in.unread(ch);
 	    if (isNumeric(ch) && ch != '-')
 	      ch = '-';
 	    else
 	      return (ttype = '-');
 	  }

 	StringBuffer tokbuf = new StringBuffer();
 	tokbuf.append((char) ch);

 	int decCount = 0;
 	while (isNumeric(ch = in.read()) && ch != '-')
 	  if (ch == '.' && decCount++ > 0)
 	    break;
 	  else
 	    tokbuf.append((char) ch);

 	in.unread(ch);
 	ttype = TT_NUMBER;
 	nval = Double.valueOf(tokbuf.toString()).doubleValue();
       }
     else if (isAlphabetic(ch))
       {
 	StringBuffer tokbuf = new StringBuffer();
 	tokbuf.append((char) ch);
 	while (isAlphabetic(ch = in.read()) || isNumeric(ch))
 	  tokbuf.append((char) ch);
 	in.unread(ch);
 	ttype = TT_WORD;
 	sval = tokbuf.toString();
 	if (lowerCase)
 	  sval.toLowerCase();
       }
     else if (isComment(ch))
       {
 	while ((ch = in.read()) != '\n' && ch != '\r' && ch != TT_EOF)
 	  ;
 	in.unread(ch);
 	return nextToken();	// Recursive, but not too deep in normal cases.
       }
     else if (isQuote(ch))
       {
 	ttype = ch;
 	StringBuffer tokbuf = new StringBuffer();
 	while ((ch = in.read()) != ttype && ch != '\n' && ch != '\r' &&
 	       ch != TT_EOF)
 	  {
 	    if (ch == '\\')
 	      switch (ch = in.read())
 		{
 		  case 'a':	ch = 0x7;
 		    break;
 		  case 'b':	ch = '\b';
 		    break;
 		  case 'f':	ch = 0xC;
 		    break;
 		  case 'n':	ch = '\n';
 		    break;
 		  case 'r':	ch = '\r';
 		    break;
 		  case 't':	ch = '\t';
 		    break;
 		  case 'v':	ch = 0xB;
 		    break;
 		  case '\"':
 		  case '\'':
 		  case '\\':
 		    break;
 		  default:
 		    int ch1, nextch;
 		    if ((nextch = ch1 = ch) >= '0' && ch <= '7')
 		      {
 		        ch -= '0';
 		        if ((nextch = in.read()) >= '0' && nextch <= '7')
 			  {
 			    ch = ch * 8 + nextch - '0';
 			    if ((nextch = in.read()) >= '0' && nextch <= '7' &&
 				ch1 >= '0' && ch1 <= '3')
 			      {
 				ch = ch * 8 + nextch - '0';
 				nextch = in.read();
 			      }
 			  }
 		      }

 		    in.unread(nextch);
 		}

 	    tokbuf.append((char) ch);
 	  }

 	// Throw away matching quote char.
 	if (ch != ttype)
 	  in.unread(ch);

 	sval = tokbuf.toString();
       }
     else
       {
         if (ch == '/')
 	  if ((ch = in.read()) == '/' && slashSlash)
 	    {
 	      while ((ch = in.read()) != '\n' && ch != '\r' && ch != TT_EOF)
 		;
 	      in.unread(ch);
 	      return nextToken(); // Recursive, but not too deep in normal cases
 	    }
 	  else if (ch == '*' && slashStar)
 	    {
 	      while (true)
 		{
 	          ch = in.read();
 		  if (ch == '*')
 		    if ((ch = in.read()) == '/')
 		      break;
 		    else
 		      in.unread(ch);
 		  else if (ch == '\n' || ch == '\r')
 		    {
 		      lineNumber++;
 		      if (ch == '\r' && (ch = in.read()) != '\n')
 			in.unread(ch);
 		    }
 		  else if (ch == TT_EOF)
 		    {
 		      in.unread(ch);
 		      break;
 		    }
 		}
 	      return nextToken(); // Recursive, but not too deep in normal cases
 	    }
 	  else
 	    {
 	      in.unread(ch);
 	      ch = '/';
 	    }

 	ttype = ch;
       }

     return ttype;
   }

   private void resetChar(int ch)
   {
     whitespace[ch] = alphabetic[ch] = numeric[ch] = quote[ch] = comment[ch] =
       false;
   }

   public void ordinaryChar(int ch)
   {
     if (ch >= 0 && ch <= 255)
       resetChar(ch);
   }

   public void ordinaryChars(int low, int hi)
   {
     if (low < 0)
       low = 0;
     if (hi > 255)
       hi = 255;
     for (int i = low; i <= hi; i++)
       resetChar(i);
   }

   public void parseNumbers()
   {
     for (int i = 0; i <= 9; i++)
       numeric['0' + i] = true;

     numeric['.'] = true;
     numeric['-'] = true;
   }

   public void pushBack()
   {
     // pushBack may cause the lineno method to return an incorrect value
     // if lineno is called before the next call to nextToken.
     pushedBack = true;
   }

   public void quoteChar(int ch)
   {
     if (ch >= 0 && ch <= 255)
       quote[ch] = true;
   }

   public void resetSyntax()
   {
     ordinaryChars(0x00, 0xFF);
   }

   public void slashSlashComments(boolean flag)
   {
     slashSlash = flag;
   }

   public void slashStarComments(boolean flag)
   {
     slashStar = flag;
   }

   public String toString()
   {
     String tempstr;
     if (ttype == TT_EOF)
       tempstr = "EOF";
     else if (ttype == TT_EOL)
       tempstr = "EOL";
     else if (ttype == TT_WORD)
       tempstr = sval;
     else if (ttype == TT_NUMBER)
       tempstr = "n=" + Double.toString(nval);
     else // must be an ordinary char.
       tempstr = "\'" + (new Character((char) ttype)).toString() + "\'";

     return "Token[" + tempstr + "], line " + Integer.toString(lineno());
   }

   public void whitespaceChars(int low, int hi)
   {
     if (low < 0)
       low = 0;
     if (hi > 255)
       hi = 255;
     for (int i = low; i <= hi; i++)
       whitespace[i] = true;
   }

   public void wordChars(int low, int hi)
   {
     if (low < 0)
       low = 0;
     if (hi > 255)
       hi = 255;
     for (int i = low; i <= hi; i++)
       alphabetic[i] = true;
   }
 }
	/* Copyright (C) 1998, 1999 Cygnus Solutions

	This file is part of libgcj.

	This software is copyrighted work licensed under the terms of the
	Libgcj License. Please consult the file "LIBGCJ_LICENSE" for
	details. */

	package java.io;

	/**
	* @author Warren Levy <warrenl@cygnus.com>
	* @date October 25, 1998.
	*/
	/* Written using "Java Class Libraries", 2nd edition, ISBN 0-201-31002-3
	* "The Java Language Specification", ISBN 0-201-63451-1
	* plus online API docs for JDK 1.2 beta from http://www.javasoft.com.
	* Status: Believed complete and correct.
	*/

	public class StreamTokenizer
	{
	/* A constant indicating that the end of the stream has been read. */
	public static final int TT_EOF = -1;

	/* A constant indicating that the end of the line has been read. */
	public static final int TT_EOL = '\n';

	/* A constant indicating that a number token has been read. */
	public static final int TT_NUMBER = -2;

	/* A constant indicating that a word token has been read. */
	public static final int TT_WORD = -3;

	/* Contains the type of the token read resulting from a call to nextToken. */
	public int ttype;

	/* The String associated with word and string tokens. */
	public String sval;

	/* The numeric value associated with number tokens. */
	public double nval;

	/* Indicates whether end-of-line is recognized as a token. */
	private boolean eolSignificant = false;

	/* Indicates whether word tokens are automatically made lower case. */
	private boolean lowerCase = false;

	/* Indicates whether C++ style comments are recognized and skipped. */
	private boolean slashSlash = false;

	/* Indicates whether C style comments are recognized and skipped. */
	private boolean slashStar = false;

	/* Attribute tables of each byte from 0x00 to 0xFF. */
	private boolean[] whitespace;
	private boolean[] alphabetic;
	private boolean[] numeric;
	private boolean[] quote;
	private boolean[] comment;

	/* The Reader associated with this class. */
	private PushbackReader in;

	/* Indicates if a token has been pushed back. */
	private boolean pushedBack = false;

	/* Contains the current line number of the reader. */
	private int lineNumber = 1;

	// Deprecated in JDK 1.1.
	public StreamTokenizer(InputStream is)
	{
	this(new InputStreamReader(is));
	}

	public StreamTokenizer(Reader r)
	{
	in = new PushbackReader(r);

	whitespace = new boolean[256];
	alphabetic = new boolean[256];
	numeric = new boolean[256];
	quote = new boolean[256];
	comment = new boolean[256];
	for (int i = 0; i < 256; i++)
	resetChar(i);

	whitespaceChars(0x00, 0x20);
	wordChars('A', 'Z');
	wordChars('a', 'z');
	wordChars(0xA0, 0xFF);
	commentChar('/');
	quoteChar('\'');
	quoteChar('"');
	parseNumbers();
	}

	public void commentChar(int ch)
	{
	if (ch >= 0 && ch <= 255)
	comment[ch] = true;
	}

	public void eolIsSignificant(boolean flag)
	{
	eolSignificant = flag;
	}

	public int lineno()
	{
	return lineNumber;
	}

	public void lowerCaseMode(boolean flag)
	{
	lowerCase = flag;
	}

	private boolean isWhitespace(int ch)
	{
	if (ch >= 0 && ch <= 255)
	return whitespace[ch];

	return false;
	}

	private boolean isAlphabetic(int ch)
	{
	if (ch >= 0 && ch <= 255)
	return alphabetic[ch];
	else if (ch > 255)
	return true;

	return false;
	}

	private boolean isNumeric(int ch)
	{
	if (ch >= 0 && ch <= 255)
	return numeric[ch];

	return false;
	}

	private boolean isQuote(int ch)
	{
	if (ch >= 0 && ch <= 255)
	return quote[ch];

	return false;
	}

	private boolean isComment(int ch)
	{
	if (ch >= 0 && ch <= 255)
	return comment[ch];

	return false;
	}

	public int nextToken() throws IOException
	{
	if (pushedBack)
	{
	pushedBack = false;
	return ttype;
	}

	sval = null;
	int ch;

	// Skip whitespace. Deal with EOL along the way.
	while (isWhitespace(ch = in.read()))
	if (ch == '\n' \|\| ch == '\r')
	{
	lineNumber++;

	// Throw away \n if in combination with \r.
	if (ch == '\r' && (ch = in.read()) != '\n')
	in.unread(ch);
	if (eolSignificant)
	return (ttype = TT_EOL);
	}

	if (ch == TT_EOF)
	ttype = TT_EOF;
	else if (isNumeric(ch))
	{
	if (ch == '-')
	{
	// Read ahead to see if this is an ordinary '-' rather than numeric.
	ch = in.read();
	in.unread(ch);
	if (isNumeric(ch) && ch != '-')
	ch = '-';
	else
	return (ttype = '-');
	}

	StringBuffer tokbuf = new StringBuffer();
	tokbuf.append((char) ch);

	int decCount = 0;
	while (isNumeric(ch = in.read()) && ch != '-')
	if (ch == '.' && decCount++ > 0)
	break;
	else
	tokbuf.append((char) ch);

	in.unread(ch);
	ttype = TT_NUMBER;
	nval = Double.valueOf(tokbuf.toString()).doubleValue();
	}
	else if (isAlphabetic(ch))
	{
	StringBuffer tokbuf = new StringBuffer();
	tokbuf.append((char) ch);
	while (isAlphabetic(ch = in.read()) \|\| isNumeric(ch))
	tokbuf.append((char) ch);
	in.unread(ch);
	ttype = TT_WORD;
	sval = tokbuf.toString();
	if (lowerCase)
	sval.toLowerCase();
	}
	else if (isComment(ch))
	{
	while ((ch = in.read()) != '\n' && ch != '\r' && ch != TT_EOF)
	;
	in.unread(ch);
	return nextToken(); // Recursive, but not too deep in normal cases.
	}
	else if (isQuote(ch))
	{
	ttype = ch;
	StringBuffer tokbuf = new StringBuffer();
	while ((ch = in.read()) != ttype && ch != '\n' && ch != '\r' &&
	ch != TT_EOF)
	{
	if (ch == '\\')
	switch (ch = in.read())
	{
	case 'a': ch = 0x7;
	break;
	case 'b': ch = '\b';
	break;
	case 'f': ch = 0xC;
	break;
	case 'n': ch = '\n';
	break;
	case 'r': ch = '\r';
	break;
	case 't': ch = '\t';
	break;
	case 'v': ch = 0xB;
	break;
	case '\"':
	case '\'':
	case '\\':
	break;
	default:
	int ch1, nextch;
	if ((nextch = ch1 = ch) >= '0' && ch <= '7')
	{
	ch -= '0';
	if ((nextch = in.read()) >= '0' && nextch <= '7')
	{
	ch = ch * 8 + nextch - '0';
	if ((nextch = in.read()) >= '0' && nextch <= '7' &&
	ch1 >= '0' && ch1 <= '3')
	{
	ch = ch * 8 + nextch - '0';
	nextch = in.read();
	}
	}
	}

	in.unread(nextch);
	}

	tokbuf.append((char) ch);
	}

	// Throw away matching quote char.
	if (ch != ttype)
	in.unread(ch);

	sval = tokbuf.toString();
	}
	else
	{
	if (ch == '/')
	if ((ch = in.read()) == '/' && slashSlash)
	{
	while ((ch = in.read()) != '\n' && ch != '\r' && ch != TT_EOF)
	;
	in.unread(ch);
	return nextToken(); // Recursive, but not too deep in normal cases
	}
	else if (ch == '*' && slashStar)
	{
	while (true)
	{
	ch = in.read();
	if (ch == '*')
	if ((ch = in.read()) == '/')
	break;
	else
	in.unread(ch);
	else if (ch == '\n' \|\| ch == '\r')
	{
	lineNumber++;
	if (ch == '\r' && (ch = in.read()) != '\n')
	in.unread(ch);
	}
	else if (ch == TT_EOF)
	{
	in.unread(ch);
	break;
	}
	}
	return nextToken(); // Recursive, but not too deep in normal cases
	}
	else
	{
	in.unread(ch);
	ch = '/';
	}

	ttype = ch;
	}

	return ttype;
	}

	private void resetChar(int ch)
	{
	whitespace[ch] = alphabetic[ch] = numeric[ch] = quote[ch] = comment[ch] =
	false;
	}

	public void ordinaryChar(int ch)
	{
	if (ch >= 0 && ch <= 255)
	resetChar(ch);
	}

	public void ordinaryChars(int low, int hi)
	{
	if (low < 0)
	low = 0;
	if (hi > 255)
	hi = 255;
	for (int i = low; i <= hi; i++)
	resetChar(i);
	}

	public void parseNumbers()
	{
	for (int i = 0; i <= 9; i++)
	numeric['0' + i] = true;

	numeric['.'] = true;
	numeric['-'] = true;
	}

	public void pushBack()
	{
	// pushBack may cause the lineno method to return an incorrect value
	// if lineno is called before the next call to nextToken.
	pushedBack = true;
	}

	public void quoteChar(int ch)
	{
	if (ch >= 0 && ch <= 255)
	quote[ch] = true;
	}

	public void resetSyntax()
	{
	ordinaryChars(0x00, 0xFF);
	}

	public void slashSlashComments(boolean flag)
	{
	slashSlash = flag;
	}

	public void slashStarComments(boolean flag)
	{
	slashStar = flag;
	}

	public String toString()
	{
	String tempstr;
	if (ttype == TT_EOF)
	tempstr = "EOF";
	else if (ttype == TT_EOL)
	tempstr = "EOL";
	else if (ttype == TT_WORD)
	tempstr = sval;
	else if (ttype == TT_NUMBER)
	tempstr = "n=" + Double.toString(nval);
	else // must be an ordinary char.
	tempstr = "\'" + (new Character((char) ttype)).toString() + "\'";

	return "Token[" + tempstr + "], line " + Integer.toString(lineno());
	}

	public void whitespaceChars(int low, int hi)
	{
	if (low < 0)
	low = 0;
	if (hi > 255)
	hi = 255;
	for (int i = low; i <= hi; i++)
	whitespace[i] = true;
	}

	public void wordChars(int low, int hi)
	{
	if (low < 0)
	low = 0;
	if (hi > 255)
	hi = 255;
	for (int i = low; i <= hi; i++)
	alphabetic[i] = true;
	}
	}