ilcclib.Tokenizer.CTokenizer.ReadNext C# (CSharp) Method

        public CToken ReadNext()
        {
            StartPos = CurrentPos;

            if (CurrentPos >= Text.Length) return new CToken()
            {
                Position = GetTokenPositionAtCurrentPos(),
                Raw = "",
                Type = CTokenType.End,
            };

            var Char = Text[CurrentPos];
            int CharactersLeft = Text.Length - CurrentPos;
            var Type = CTokenType.None;
            bool DoIsNumber = false;

            if (IsNumber(Char) || (Char == '.' && IsNumber(Text[CurrentPos + 1])))
            {
                DoIsNumber = true;
            }

            // Skip spaces
            if (IsSpace(Char))
            {
                if (TokenizeSpaces)
                {
                    if (Char == '\n')
                    {
                        CurrentPos++;
                        Type = CTokenType.NewLine;
                    }
                    else
                    {
                        for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
                        {
                            var Char2 = Text[CurrentPos];
                            if (!IsSpace(Char2) || Char2 == '\n') break;
                        }
                        Type = CTokenType.Space;
                    }
                }
                else
                {
                    while (CurrentPos < Text.Length)
                    {
                        Char = Text[CurrentPos];
                        if (!IsSpace(Char)) break;
                        CurrentPos++;
                    }
                    return ReadNext();
                }
            }
            // Number?
            else if (DoIsNumber)
            {
                var NumStr = "" + Char;
                // TODO: support exponent and float numbers.
                for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
                {
                    var Char2 = Text[CurrentPos];
                    if (!IsNumberContinuation(Char2)) break;
                    NumStr += Char2;
                }
                bool IsDouble = (NumStr.Contains("."));
                if (NumStr.EndsWith("f")) IsDouble = true;
                if (NumStr.ToLower().StartsWith("0x")) IsDouble = false;
                //Console.WriteLine(NumStr);
                Type = IsDouble ? CTokenType.Float : CTokenType.Integer;
            }
            // Identifier?
            else if (IsFirstIdentifier(Char))
            {
                for (CurrentPos++; CurrentPos < Text.Length; CurrentPos++)
                {
                    var Char2 = Text[CurrentPos];
                    if (!IsAnyIdentifier(Char2)) break;
                }
                Type = CTokenType.Identifier;
            }
            // ' or "
            else if (Char == '\'' || Char == '"')
            {
                bool Closed = false;
                CurrentPos++;

                for (; CurrentPos < Text.Length; CurrentPos++)
                {
                    var Char2 = Text[CurrentPos];
                    if (Char2 == '\\') { CurrentPos++; continue; }
                    if (Char2 == Char) { Closed = true; break; }
                }

                if (!Closed) throw (new Exception(String.Format("Not closed string/char : [[ {0} ]]", Text.Substring(StartPos))));

                CurrentPos++;
                Type = (Char == '\'') ? CTokenType.Char : CTokenType.String;
            }
            // Operators
            else
            {
                if (CharactersLeft >= 3 && Operators3.Contains(Text.Substring(CurrentPos, 3))) CurrentPos += 3;
                else if (CharactersLeft >= 2 && Operators2.Contains(Text.Substring(CurrentPos, 2))) CurrentPos += 2;
                else if (CharactersLeft >= 1 && Operators1.Contains(Text.Substring(CurrentPos, 1))) CurrentPos += 1;
                else
                {
                    throw (new NotImplementedException(String.Format("Unknown character '{0}'", Char)));
                }
                Type = CTokenType.Operator;
            }

            if (!(CurrentPos > StartPos && StartPos < Text.Length && CurrentPos <= Text.Length))
            {
                Console.Error.WriteLine(Text);
                throw (new Exception("Invalid position"));
            }
            var Token = new CToken()
            {
                Position = GetTokenPositionAtCurrentPos(),
                Raw = Text.Substring(StartPos, CurrentPos - StartPos),
                Type = Type,
            };

            return Token;
        }