public TokenType NextToken()
{
SkipComments();
// Clear the TokenType
_builder.Length = 0;
_token = String.Empty;
_tokenLine = _line;
_tokenLinePos = _linePos;
if (!_nextEOF)
{
if (Char.IsLetter(_next) || (_next == '_')) // Identifiers
{
Advance();
_builder.Append(_current);
while (!_nextEOF && (Char.IsLetterOrDigit(_next) || (_next == '_')))
{
Advance();
_builder.Append(_current);
}
_token = _builder.ToString();
_tokenType = TokenType.Symbol;
}
else if (IsSymbol(_next)) // Symbols
{
_tokenType = TokenType.Symbol;
Advance();
_builder.Append(_current);
switch (_current)
{
case ':':
if (!_nextEOF && (_next == '='))
{
Advance();
_builder.Append(_current);
}
break;
case '?':
if (!_nextEOF)
switch (_next)
{
case '=':
case '?':
Advance();
_builder.Append(_current);
break;
}
break;
case '*':
if (!_nextEOF && (_next == '*'))
{
Advance();
_builder.Append(_current);
}
break;
case '>':
if (!_nextEOF)
switch (_next)
{
case '=':
case '>':
Advance();
_builder.Append(_current);
break;
}
break;
case '<':
if (!_nextEOF)
switch (_next)
{
case '=':
case '>':
case '<':
Advance();
_builder.Append(_current);
break;
}
break;
case '=':
if (!_nextEOF && _next == '>')
{
Advance();
_builder.Append(_current);
break;
}
break;
case '-':
if (!_nextEOF && _next == '>')
{
Advance();
_builder.Append(_current);
break;
}
break;
case '.':
if (!_nextEOF && _next == '.')
{
Advance();
_builder.Append(_current);
break;
}
break;
}
_token = _builder.ToString();
}
else if (Char.IsDigit(_next)) // Numbers
{
Advance();
bool digitSatisfied = false; // at least one digit required
if ((_current == '0') && (!_nextEOF && ((_next == 'x') || (_next == 'X'))))
{
_tokenType = TokenType.Hex;
Advance();
}
else
{
digitSatisfied = true;
_builder.Append(_current);
_tokenType = TokenType.Integer;
}
bool done = false;
int periodsHit = 0;
bool hitScalar = false;
while (!done && !_nextEOF)
{
switch (_next)
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
if (_tokenType == TokenType.Integer && ExceedsInt32(_builder))
_tokenType = TokenType.Long;
else if (_tokenType == Parse.TokenType.Hex && ExceedsInt32Hex(_builder))
_tokenType = TokenType.LongHex;
Advance();
digitSatisfied = true;
_builder.Append(_current);
break;
case 'A':
case 'a':
case 'B':
case 'b':
case 'C':
case 'c':
case 'D':
case 'd':
case 'f':
case 'F':
if (_tokenType != TokenType.Hex && _tokenType != TokenType.LongHex)
{
done = true;
break;
}
Advance();
digitSatisfied = true;
_builder.Append(_current);
break;
case 'E':
case 'e':
if ((_tokenType != TokenType.Hex && _tokenType != TokenType.LongHex) || (!hitScalar && digitSatisfied))
{
done = true;
break;
}
Advance();
if (_tokenType == TokenType.Hex || _tokenType == TokenType.LongHex)
{
digitSatisfied = true;
_builder.Append(_current);
}
else if (!hitScalar && digitSatisfied)
{
hitScalar = true;
digitSatisfied = false;
_tokenType = TokenType.Double;
_builder.Append(_current);
if ((_next == '-') || (_next == '+'))
{
Advance();
_builder.Append(_current);
}
}
break;
case '.':
// If interval, or we've fully satisfied a version, stop looking for digits
if ((!PeekEOF && Peek == '.') || periodsHit >= 3)
{
done = true;
break;
}
if (!digitSatisfied || hitScalar)
throw new LexerException(LexerException.Codes.InvalidNumericValue);
if (periodsHit == 0)
{
if (_tokenType == TokenType.Hex || _tokenType == TokenType.LongHex)
throw new LexerException(LexerException.Codes.InvalidNumericValue);
if (_tokenType == TokenType.Integer)
_tokenType = TokenType.Double;
}
else if (periodsHit == 1)
_tokenType = TokenType.Version;
Advance();
_builder.Append(_current);
periodsHit++;
digitSatisfied = false;
break;
default:
done = true;
break;
}
}
if (!digitSatisfied)
throw new LexerException(LexerException.Codes.InvalidNumericValue);
_token = _builder.ToString();
}
else if (_next == '"') // C-style string
{
_tokenType = TokenType.String;
Advance();
var terminated = false;
while (!terminated)
{
if (_nextEOF)
throw new LexerException(LexerException.Codes.UnterminatedString); // Better error than EOF
Advance();
switch (_current)
{
case '\\' :
if (_nextEOF)
throw new LexerException(LexerException.Codes.InvalidEscapeCharacter);
switch (_next)
{
case '\\' :
case '\"' : _builder.Append(_next); Advance(); break;
case 'n' : _builder.Append('\n'); Advance(); break;
case 'r' : _builder.Append('\r'); Advance(); break;
case 't' : _builder.Append('\t'); Advance(); break;
default: throw new LexerException(LexerException.Codes.InvalidEscapeCharacter, _next);
}
break;
case '\"' :
terminated = true;
break;
default :
_builder.Append(_current);
break;
}
}
_token = _builder.ToString();
}
else if (_next == '\'') // Pascal-style string
{
_tokenType = TokenType.String;
Advance();
while (true)
{
if (_nextEOF)
throw new LexerException(LexerException.Codes.UnterminatedString); // Better error than EOF
Advance();
if (_current == '\'')
{
if (!_nextEOF && _next == '\'')
Advance();
else
break;
}
_builder.Append(_current);
}
if (!_nextEOF)
switch (_next)
{
case 'c' :
Advance();
_tokenType = TokenType.Char;
if (_builder.Length != 1)
throw new LexerException(LexerException.Codes.InvalidCharacter);
break;
case 'n' :
Advance();
_tokenType = TokenType.Name;
break;
case 'd' :
Advance();
if (!_nextEOF && _next == 't')
{
Advance();
_tokenType = TokenType.DateTime;
}
else
_tokenType = TokenType.Date;
break;
case 't' :
Advance();
if (!_nextEOF && _next == 's')
{
Advance();
_tokenType = TokenType.TimeSpan;
}
else
_tokenType = TokenType.Time;
break;
case 'g' :
Advance();
_tokenType = TokenType.Guid;
break;
}
_token = _builder.ToString();
}
else if (_next == '#')
{
_tokenType = TokenType.String;
Advance();
var digitSatisfied = false;
int code = 0;
while (!_nextEOF && Char.IsDigit(_next))
{
digitSatisfied = true;
Advance();
code = code * 10 + (_current - '0');
}
if (!digitSatisfied)
throw new LexerException(LexerException.Codes.InvalidCharacterCode);
_token = Convert.ToChar(code).ToString();
}
else
throw new LexerException(LexerException.Codes.IllegalInputCharacter, _next);
}
else
_tokenType = TokenType.EOF;
return _tokenType;
}
