Mono.Rocks.TextReaderRocks.CreateTokensIterator C# (CSharp) Method

        private static IEnumerable<string> CreateTokensIterator(TextReader self, TextReaderRocksOptions options, Func<char?, char, bool>[] categories)
        {
            try {
                var cats = categories.Select (
                        c => Lambda.F ((StringBuilder buf, char ch) =>
                            c (buf.Length == 0 ? ((char?) null) : (char?) buf [buf.Length-1], ch)));
                foreach (var t in Chars (self).Tokens (
                            new StringBuilder (),
                            (buf, c) => buf.Append (c),
                            buf => {
                                var r = buf.ToString ();
                                buf.Length = 0;
                                return Tuple.Create (r, buf);
                            },
                            cats.ToArray ()))
                    yield return t;
            } finally {
                if ((options & TextReaderRocksOptions.CloseReader) != 0) {
                    self.Close ();
                    self.Dispose ();
                }
            }
        }