AjTalk.Compiler.Lexer.NextToken C# (CSharp) Method

        public Token NextToken()
        {
            if (this.tokenstack.Count > 0)
            {
                return this.tokenstack.Pop();
            }

            int ch;

            ch = this.NextCharSkipBlanksAndComments();

            if (ch < 0)
                return null;

            if (char.IsLetter((char)ch) || ch == '_')
            {
                return this.NextName((char)ch);
            }

            if (char.IsDigit((char)ch))
            {
                return this.NextInteger((char)ch);
            }

            if (ch == StringDelimiter)
            {
                return this.NextString();
            }

            if (ch == SpecialCharMark)
            {
                int ch2 = this.NextChar();
                return new Token() { Type = TokenType.Character, Value = ((char)ch2).ToString() };
            }

            if (ch == SymbolMark)
            {
                int ch2 = this.NextChar();

                if (ch2 == '(')
                    return new Token() { Type = TokenType.Punctuation, Value = "#(" };
                if (ch2 == '[')
                    return new Token() { Type = TokenType.Punctuation, Value = "#[" };
                if (ch2 == '{')
                    return this.NextEnclosedSymbol();
                if (ch2 == '\'')
                    return new Token() { Type = TokenType.Symbol, Value = this.NextString().Value };

                this.PushChar(ch2);

                return this.NextSymbol();
            }

            if (ch == ParameterMark)
            {
                int ch2 = this.PeekChar();

                if (ch2 < 0 || !char.IsLetter((char)ch2))
                {
                    return this.NextOperator((char)ch);
                }

                return this.NextParameter();
            }

            if (ch == SpecialDotNetTypeMark)
            {
                int ch2 = this.PeekChar();

                if (ch2 < 0 || !char.IsLetter((char)ch2) || !char.IsUpper((char)ch2))
                {
                    return this.NextOperator((char)ch);
                }

                return this.NextDotNetTypeName();
            }

            if (ch == SpecialDotNetInvokeMark)
            {
                return this.NextDotNetInvokeName();
            }

            if (Operators.IndexOf((char)ch) >= 0)
            {
                return this.NextOperator((char)ch);
            }

            if (Separators.IndexOf((char)ch) >= 0)
            {
                return this.NextPunctuation((char)ch);
            }

            throw new LexerException("Invalid Characater '" + (char)ch + "'");
        }

        public void ParseDotNetMethodMethodWithParenthesis()
        {
            Lexer tokenizer = new Lexer("(anObject !nativeMethod)");
            Token token;

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("(", token.Value);
            Assert.AreEqual(TokenType.Punctuation, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("anObject", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual("!nativeMethod", token.Value);
            Assert.AreEqual(TokenType.Name, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNotNull(token);
            Assert.AreEqual(")", token.Value);
            Assert.AreEqual(TokenType.Punctuation, token.Type);

            token = tokenizer.NextToken();
            Assert.IsNull(token);
        }