Kaleidoscope.Chapter6.Lexer.GetToken C# (CSharp) Method

        public Token GetToken()
        {
            if (m_bEof || !AdvanceReader())
            {
                m_bEof = false;
                return new Token(TokenCode.Eof);
            }

            char ch = m_line[m_position];

            // identifier: [a-zA-Z][a-zA-Z0-9]*
            if(char.IsLetter(ch)) 
            {
                StringBuilder sb = new StringBuilder();
                do
                {
                    sb.Append(ch);
                    ch = NextChar();
                }
                while(char.IsLetterOrDigit(ch));

                string identifier = sb.ToString();
                switch(identifier)
                {
                    case "def":
                        return new Token(TokenCode.Def);
                    case "extern":
                        return new Token(TokenCode.Extern);
                    case "if":
                        return new Token(TokenCode.If);
                    case "then":
                        return new Token(TokenCode.Then);
                    case "else":
                        return new Token(TokenCode.Else);
                    case "for":
                        return new Token(TokenCode.For);
                    case "in":
                        return new Token(TokenCode.In);
                    case "binary":
                        return new Token(TokenCode.Binary);
                    case "unary":
                        return new Token(TokenCode.Unary);
                }

                return new Token(identifier); 
            }

            // Number: [0-9.]+
            if(char.IsNumber(ch) || ch == '.')
            {
                StringBuilder sb = new StringBuilder();
                do
                {
                    sb.Append(ch);
                    ch = NextChar();
                }
                while(char.IsDigit(ch) || ch == '.');

                return new Token(double.Parse(sb.ToString()));
            }

            if(ch == '#')
            {
                // Comment until end of line.
                m_position = m_line.Length;
                return GetToken();
            }

            m_position++;

            return new Token(ch);
        }