public Token Next()
{
retry:
if (_eof())
return null;
char c = _peekChar();
if (Char.IsWhiteSpace(c))
{
_skipWhiteSpace();
goto retry;
}
if(c == '–')
{
// long dash, skip
_readChar();
goto retry;
}
if(Char.IsSymbol(c) || Char.IsPunctuation(c))
{
string symbol = _readSymbolString();
return _resolve(_symbolTypes, symbol, true);
}
if(Char.IsLetter(c))
{
string ident = _readIdentifierString();
var token = _resolve(_identifierTypes, ident, false);
if (token == null)
token = new Token(TokenType.Identifier, ident);
return token;
}
if (Char.IsDigit(c))
{
string number = _readInteger();
return new Token(TokenType.Integer, number);
}
throw new InvalidDataException(c.ToString());
}