Lucene.Net.Analysis.Synonym.SlowSynonymFilterFactory.SplitByTokenizer C# (CSharp) Метод

        private static IList<string> SplitByTokenizer(string source, TokenizerFactory tokFactory)
        {
            StringReader reader = new StringReader(source);
            TokenStream ts = LoadTokenizer(tokFactory, reader);
            IList<string> tokList = new List<string>();
            try
            {
                ICharTermAttribute termAtt = ts.AddAttribute<ICharTermAttribute>();
                ts.Reset();
                while (ts.IncrementToken())
                {
                    if (termAtt.Length > 0)
                    {
                        tokList.Add(termAtt.ToString());
                    }
                }
            }
            finally
            {
                reader.Dispose();
            }
            return tokList;
        }