ilcclib.Tokenizer.CTokenizer.Tokenize C# (CSharp) Метод

        public IEnumerable<CToken> Tokenize()
        {
            CToken LastToken;
            //if (Text.Length > 0)
            do
            {
                LastToken = ReadNext();
                yield return LastToken;
            } while (LastToken.Type != CTokenType.End);
        }

 public void TestTokenize4()
 {
     var CTokenizer = new CTokenizer("test\n  #include", TokenizeSpaces: false);
     var Tokens = CTokenizer.Tokenize().ToArray();
     Assert.AreEqual("Position:0, Row:0, Column:0, ColumnNoSpaces:0", Tokens[0].Position.ToString());
     Assert.AreEqual("Position:7, Row:1, Column:2, ColumnNoSpaces:0", Tokens[1].Position.ToString());
     Assert.AreEqual("Position:8, Row:1, Column:3, ColumnNoSpaces:1", Tokens[2].Position.ToString());
 }