Acquarella.Lexers.Lexer.GetTokens C# (CSharp) Метод

        public IEnumerable<Token> GetTokens(string text)
        {
            this.position = 0;
            this.text = text;
            this.length = text.Length;

            Token token;

            while ((token = this.GetNextToken()) != null)
                yield return token;
        }

        public IEnumerable<TokenType> GetColorTypes(string text, Lexer lexer)
        {
            int position = 0;

            foreach (var token in lexer.GetTokens(text))
            {
                while (position < token.Start)
                {
                    yield return TokenType.Space;
                    position++;
                }

                while (position < token.Start + token.Length)
                {
                    yield return token.Type;
                    position++;
                }
            }

            int length = text.Length;

            while (position < length)
            {
                yield return TokenType.Space;
                position++;
            }
        }