tools/tilemanager/Lexer.cs

   1 //  $Id$
   2 using System;
   3 using System.Text;
   4 using System.IO;
   5
   6 namespace Lisp {
   7
   8 public class Lexer {
   9     private StreamReader stream;
  10     private char[] buffer;
  11     private char c;
  12     int bufpos;
  13     int buflen;
  14
  15     public class EOFException : Exception {
  16     };
  17
  18     public enum TokenType {
  19         EOF,
  20         OPEN_PAREN,
  21         CLOSE_PAREN,
  22         SYMBOL,
  23         STRING,
  24         INTEGER,
  25         REAL,
  26         TRUE,
  27         FALSE
  28     };
  29
  30     private StringBuilder TokenStringBuilder;
  31     public string TokenString {
  32         get { return TokenStringBuilder.ToString(); }
  33     }
  34     public int LineNumber;
  35
  36     public Lexer(StreamReader stream) {
  37         this.stream = stream;
  38         buffer = new char[1025];
  39         NextChar();
  40     }
  41
  42     public TokenType GetNextToken() {
  43         try {
  44             while(Char.IsWhiteSpace(c)) {
  45                 NextChar();
  46                 if(c == '\n')
  47                     LineNumber++;
  48             }
  49
  50             TokenStringBuilder = new StringBuilder();
  51
  52             switch(c) {
  53                 case ';': // comment
  54                     while(true) {
  55                         NextChar();
  56                         if(c == '\n') {
  57                             LineNumber++;
  58                             break;
  59                         }
  60                     }
  61                     NextChar();
  62                     return GetNextToken();
  63                 case '(':
  64                     NextChar();
  65                     return TokenType.OPEN_PAREN;
  66                 case ')':
  67                     NextChar();
  68                     return TokenType.CLOSE_PAREN;
  69                 case '"': { // string
  70                     int startline = LineNumber;
  71                     while(true) {
  72                         NextChar();
  73                         if(c == '"')
  74                             break;
  75
  76                         if(c == '\\') {
  77                             NextChar();
  78                             switch(c) {
  79                                 case 'n':
  80                                     c = '\n';
  81                                     break;
  82                                 case 't':
  83                                     c = '\t';
  84                                     break;
  85                             }
  86                         }
  87                         TokenStringBuilder.Append(c);
  88                     }
  89                     NextChar();
  90                     return TokenType.STRING;
  91                 }
  92                 case '#': // constant
  93                     NextChar();
  94                     while(Char.IsLetterOrDigit(c) || c == '_') {
  95                         TokenStringBuilder.Append(c);
  96                         NextChar();
  97                     }
  98                     if(TokenString == "t")
  99                         return TokenType.TRUE;
 100                     if(TokenString == "f")
 101                         return TokenType.FALSE;
 102
 103                     throw new Exception("Unknown constant '"
 104                             + TokenString + "'");
 105                 default:
 106                     if(Char.IsDigit(c) || c == '-') {
 107                         bool have_nondigits = false;
 108                         bool have_digits = false;
 109                         int have_floating_point = 0;
 110
 111                         do {
 112                             if(Char.IsDigit(c))
 113                                 have_digits = true;
 114                             else if(c == '.')
 115                                 have_floating_point++;
 116                             else if(Char.IsLetter(c) || c == '_')
 117                                 have_nondigits = true;
 118
 119                             TokenStringBuilder.Append(c);
 120                             NextChar();
 121                         } while(!Char.IsWhiteSpace(c) && c != '\"' && c != '('
 122                                 && c != ')' && c != ';');
 123
 124                         if(have_nondigits || !have_digits
 125                                 || have_floating_point > 1)
 126                             return TokenType.SYMBOL;
 127                         else if(have_floating_point == 1)
 128                             return TokenType.REAL;
 129                         else
 130                             return TokenType.INTEGER;
 131                     } else {
 132                         do {
 133                             TokenStringBuilder.Append(c);
 134                             NextChar();
 135                         } while(!Char.IsWhiteSpace(c) && c != '\"' && c != '('
 136                                 && c != ')' && c != ';');
 137
 138                         return TokenType.SYMBOL;
 139                     }
 140             }
 141         } catch(EOFException) {
 142             return TokenType.EOF;
 143         }
 144     }
 145
 146     private void NextChar() {
 147         if(bufpos >= buflen) {
 148             if(!stream.BaseStream.CanRead)
 149                 throw new EOFException();
 150             buflen = stream.Read(buffer, 0, 1024);
 151             bufpos = 0;
 152             // following hack appends an additional ' ' at the end of the file
 153             // to avoid problems when parsing symbols/elements and a sudden EOF:
 154             // This way we can avoid the need for an unget function.
 155             if(!stream.BaseStream.CanRead) {
 156                 buffer[buflen] = ' ';
 157                 ++buflen;
 158             }
 159         }
 160         c = buffer[bufpos++];
 161     }
 162 }
 163
 164 }