Alsing.Text.Tokenizer.AddPattern C# (CSharp) Method

        public Tokenizer AddPattern(IPatternMatcher matcher,bool caseSensitive,bool needsSeparators,params object[] tags)
        {
            ThrowIfImmutable();

            tree.AddPattern(matcher, caseSensitive, needsSeparators, tags);
            return this;
        }

        public void ParsePatternWithoutSeparators()
        {
            var tokenizer = new Tokenizer();

            tokenizer.AddPattern(IntPatternMatcher.Default, true, false, TestTag);

            tokenizer.AddToken("Alsing", false, true, TestTag);

            const string text = @"The quick brown 1337 fox jumped 0v3r the little pig";
            //                                    XXXX            X X

            // only two tokens should be found, the rest
            // are either wrong casing or do not have separators next to them

            tokenizer.Text = text;
            var tokens = tokenizer.Tokenize();

            var testTokens = from token in tokens
                             where token.HasTag(TestTag)
                             select token;

            Assert.AreEqual(3, testTokens.ToList().Count);
        }