Alsing.Text.Tokenizer.Tokenize C# (CSharp) Method

        public Token[] Tokenize()
        {
            if (Text == null)
                throw new ArgumentNullException("Text");

            MakeImmutable();

            var tokens = new List<Token>();

            int index = 0;
            while (index < Text.Length)
            {
                MatchResult match = tree.Match(Text, index);

                if (match.Found)
                {
                    string dummyText = Text.Substring(index, match.Index - index);
                    var dummyToken = new Token(dummyText, null);
                    tokens.Add(dummyToken);

                    var realToken = new Token(match.GetText(), match.Tags);
                    index = match.Index + match.Length;
                    tokens.Add(realToken);                    
                }
                else
                {
                    string dummyText = Text.Substring(index);
                    var dummyToken = new Token(dummyText, null);
                    tokens.Add(dummyToken);

                    index = Text.Length;
                }
            }

            return tokens.ToArray();
        }

        public void ParseCaseInsensitiveTokensWithoutSeparators()
        {
            var tokenizer = new Tokenizer();

            tokenizer.AddToken("Roger", false, false, TestTag);
            tokenizer.AddToken("Alsing", false, false, TestTag);

            const string text = @"Roger ROGER rogeR Alsing ALSING __Roger__ __Alsing__";
            //                   "XXXXX XXXXX XXXXX XXXXXX XXXXXX   XXXXX     XXXXXX  "

            // only two tokens should be found, the rest
            // are either wrong casing or do not have separators next to them

            tokenizer.Text = text;
            var tokens = tokenizer.Tokenize();

            var testTokens = from token in tokens
                             where token.HasTag(TestTag)
                             select token;

            Assert.AreEqual(7, testTokens.ToList().Count);
        }