Alsing.Text.Tokenizer.AddToken C# (CSharp) Method

        public Tokenizer AddToken(string token, bool caseSensitive, bool needsSeparators, params object[] tags)
        {
            ThrowIfImmutable();

            tree.AddToken(token,caseSensitive, needsSeparators, tags);
            return this;
        }

        public void ParseCaseSensitiveTokens()
        {
            var tokenizer = new Tokenizer();

            tokenizer.AddToken("Roger", true, true, TestTag);
            tokenizer.AddToken("Alsing", true, true, TestTag);

            const string text = @"Roger ROGER rogeR Alsing ALSING __Roger__ __Alsing__";
            //                   "XXXXX             XXXXXX                            "

            // only two tokens should be found, the rest
            // are either wrong casing or do not have separators next to them

            tokenizer.Text = text;
            var tokens = tokenizer.Tokenize();

            var testTokens = from token in tokens
                             where token.HasTag(TestTag)
                             select token;

            Assert.AreEqual(2, testTokens.ToList().Count);
        }