tools/tilemanager/Lexer.cs

   1 using System;
   2 using System.Text;
   3 using System.IO;
   4
   5 namespace Lisp {
   6
   7 public class Lexer {
   8     private StreamReader stream;
   9     private char[] buffer;
  10     private char c;
  11     int bufpos;
  12     int buflen;
  13
  14     public class EOFException : Exception {
  15     };
  16
  17     public enum TokenType {
  18         EOF,
  19         OPEN_PAREN,
  20         CLOSE_PAREN,
  21         SYMBOL,
  22         STRING,
  23         INTEGER,
  24         REAL,
  25         TRUE,
  26         FALSE
  27     };
  28
  29     private StringBuilder TokenStringBuilder;
  30     public string TokenString {
  31         get { return TokenStringBuilder.ToString(); }
  32     }
  33     public int LineNumber;
  34
  35     public Lexer(StreamReader stream) {
  36         this.stream = stream;
  37         buffer = new char[1025];
  38         NextChar();
  39     }
  40
  41     public TokenType GetNextToken() {
  42         try {
  43             while(Char.IsWhiteSpace(c)) {
  44                 NextChar();
  45                 if(c == '\n')
  46                     LineNumber++;
  47             }
  48
  49             TokenStringBuilder = new StringBuilder();
  50
  51             switch(c) {
  52                 case ';': // comment
  53                     while(true) {
  54                         NextChar();
  55                         if(c == '\n') {
  56                             LineNumber++;
  57                             break;
  58                         }
  59                     }
  60                     NextChar();
  61                     return GetNextToken();
  62                 case '(':
  63                     NextChar();
  64                     return TokenType.OPEN_PAREN;
  65                 case ')':
  66                     NextChar();
  67                     return TokenType.CLOSE_PAREN;
  68                 case '"': { // string
  69                     int startline = LineNumber;
  70                     while(true) {
  71                         NextChar();
  72                         if(c == '"')
  73                             break;
  74
  75                         if(c == '\\') {
  76                             NextChar();
  77                             switch(c) {
  78                                 case 'n':
  79                                     c = '\n';
  80                                     break;
  81                                 case 't':
  82                                     c = '\t';
  83                                     break;
  84                             }
  85                         }
  86                         TokenStringBuilder.Append(c);
  87                     }
  88                     NextChar();
  89                     return TokenType.STRING;
  90                 }
  91                 case '#': // constant
  92                     NextChar();
  93                     while(Char.IsLetterOrDigit(c) || c == '_') {
  94                         TokenStringBuilder.Append(c);
  95                         NextChar();
  96                     }
  97                     if(TokenString == "t")
  98                         return TokenType.TRUE;
  99                     if(TokenString == "f")
 100                         return TokenType.FALSE;
 101
 102                     throw new Exception("Unknown constant '"
 103                             + TokenString + "'");
 104                 default:
 105                     if(Char.IsDigit(c) || c == '-') {
 106                         bool have_nondigits = false;
 107                         bool have_digits = false;
 108                         int have_floating_point = 0;
 109
 110                         do {
 111                             if(Char.IsDigit(c))
 112                                 have_digits = true;
 113                             else if(c == '.')
 114                                 have_floating_point++;
 115                             else if(Char.IsLetter(c) || c == '_')
 116                                 have_nondigits = true;
 117
 118                             TokenStringBuilder.Append(c);
 119                             NextChar();
 120                         } while(!Char.IsWhiteSpace(c) && c != '\"' && c != '('
 121                                 && c != ')' && c != ';');
 122
 123                         if(have_nondigits || !have_digits
 124                                 || have_floating_point > 1)
 125                             return TokenType.SYMBOL;
 126                         else if(have_floating_point == 1)
 127                             return TokenType.REAL;
 128                         else
 129                             return TokenType.INTEGER;
 130                     } else {
 131                         do {
 132                             TokenStringBuilder.Append(c);
 133                             NextChar();
 134                         } while(!Char.IsWhiteSpace(c) && c != '\"' && c != '('
 135                                 && c != ')' && c != ';');
 136
 137                         return TokenType.SYMBOL;
 138                     }
 139             }
 140         } catch(EOFException) {
 141             return TokenType.EOF;
 142         }
 143     }
 144
 145     private void NextChar() {
 146         if(bufpos >= buflen) {
 147             if(!stream.BaseStream.CanRead)
 148                 throw new EOFException();
 149             buflen = stream.Read(buffer, 0, 1024);
 150             bufpos = 0;
 151             // following hack appends an additional ' ' at the end of the file
 152             // to avoid problems when parsing symbols/elements and a sudden EOF:
 153             // This way we can avoid the need for an unget function.
 154             if(!stream.BaseStream.CanRead) {
 155                 buffer[buflen] = ' ';
 156                 ++buflen;
 157             }
 158         }
 159         c = buffer[bufpos++];
 160     }
 161 }
 162
 163 }