private static IList<string> SplitByTokenizer(string source, TokenizerFactory tokFactory)
{
StringReader reader = new StringReader(source);
TokenStream ts = LoadTokenizer(tokFactory, reader);
IList<string> tokList = new List<string>();
try
{
ICharTermAttribute termAtt = ts.AddAttribute<ICharTermAttribute>();
ts.Reset();
while (ts.IncrementToken())
{
if (termAtt.Length > 0)
{
tokList.Add(termAtt.ToString());
}
}
}
finally
{
reader.Dispose();
}
return tokList;
}