public CToken ReadNext()
{
StartPos = CurrentPos;
if (CurrentPos >= Text.Length) return new CToken()
{
Position = GetTokenPositionAtCurrentPos(),
Raw = "",
Type = CTokenType.End,
};
var Char = Text[CurrentPos];
int CharactersLeft = Text.Length - CurrentPos;
var Type = CTokenType.None;
bool DoIsNumber = false;
if (IsNumber(Char) || (Char == '.' && IsNumber(Text[CurrentPos + 1])))
{
DoIsNumber = true;
}
// Skip spaces
if (IsSpace(Char))
{
if (TokenizeSpaces)
{
if (Char == '\n')
{
CurrentPos++;
Type = CTokenType.NewLine;
}
else
{
for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (!IsSpace(Char2) || Char2 == '\n') break;
}
Type = CTokenType.Space;
}
}
else
{
while (CurrentPos < Text.Length)
{
Char = Text[CurrentPos];
if (!IsSpace(Char)) break;
CurrentPos++;
}
return ReadNext();
}
}
// Number?
else if (DoIsNumber)
{
var NumStr = "" + Char;
// TODO: support exponent and float numbers.
for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (!IsNumberContinuation(Char2)) break;
NumStr += Char2;
}
bool IsDouble = (NumStr.Contains("."));
if (NumStr.EndsWith("f")) IsDouble = true;
if (NumStr.ToLower().StartsWith("0x")) IsDouble = false;
//Console.WriteLine(NumStr);
Type = IsDouble ? CTokenType.Float : CTokenType.Integer;
}
// Identifier?
else if (IsFirstIdentifier(Char))
{
for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (!IsAnyIdentifier(Char2)) break;
}
Type = CTokenType.Identifier;
}
// ' or "
else if (Char == '\'' || Char == '"')
{
bool Closed = false;
CurrentPos++;
for (; CurrentPos < Text.Length; CurrentPos++)
{
var Char2 = Text[CurrentPos];
if (Char2 == '\\') { CurrentPos++; continue; }
if (Char2 == Char) { Closed = true; break; }
}
if (!Closed) throw (new Exception(String.Format("Not closed string/char : [[ {0} ]]", Text.Substring(StartPos))));
CurrentPos++;
Type = (Char == '\'') ? CTokenType.Char : CTokenType.String;
}
// Operators
else
{
if (CharactersLeft >= 3 && Operators3.Contains(Text.Substring(CurrentPos, 3))) CurrentPos += 3;
else if (CharactersLeft >= 2 && Operators2.Contains(Text.Substring(CurrentPos, 2))) CurrentPos += 2;
else if (CharactersLeft >= 1 && Operators1.Contains(Text.Substring(CurrentPos, 1))) CurrentPos += 1;
else
{
throw (new NotImplementedException(String.Format("Unknown character '{0}'", Char)));
}
Type = CTokenType.Operator;
}
if (!(CurrentPos > StartPos && StartPos < Text.Length && CurrentPos <= Text.Length))
{
Console.Error.WriteLine(Text);
throw (new Exception("Invalid position"));
}
var Token = new CToken()
{
Position = GetTokenPositionAtCurrentPos(),
Raw = Text.Substring(StartPos, CurrentPos - StartPos),
Type = Type,
};
return Token;
}