AjRools.Expert.Compiler.Lexer.NextToken C# (CSharp) Метод

        public Token NextToken()
        {
            int ich = this.NextChar();

            while (ich != -1 && char.IsWhiteSpace((char)ich))
            {
                if (((char)ich) == '\n')
                    return new Token("\n", TokenType.EndOfLine);

                ich = this.NextChar();
            }

            if (ich == -1)
                return null;

            char ch = (char)ich;

            if (ch == '"')
                return this.NextString();

            if (char.IsLetter(ch))
                return this.NextName(ch);

            if (char.IsDigit(ch))
                return this.NextInteger(ch);

            if (operators.Any(op => op[0] == ch))
                return this.NextOperator(ch);

            if (separators.Any(sp => sp[0] == ch))
                return this.NextSeparator(ch);

            throw new LexerException(string.Format("Unexpected '{0}'", ch));
        }

        public void GetCarriageReturnNewLineAsEndOfLine()
        {
            Lexer lexer = new Lexer("\r\n");
            Token token = lexer.NextToken();

            Assert.IsNotNull(token);
            Assert.AreEqual(TokenType.EndOfLine, token.Type);

            Assert.IsNull(lexer.NextToken());
        }