public override bool IncrementToken()
{
// get the next piece of input
if (savedState != null)
{
RestoreState(savedState);
savedState = null;
SaveTermBuffer();
return true;
}
else if (!input.IncrementToken())
{
return false;
}
/* We build n-grams before and after stopwords.
* When valid, the buffer always contains at least the separator.
* If its empty, there is nothing before this stopword.
*/
if (lastWasCommon || (Common && buffer.Length > 0))
{
savedState = CaptureState();
GramToken();
return true;
}
SaveTermBuffer();
return true;
}
public virtual void TestReset()
{
const string input = "How the s a brown s cow d like A B thing?";
WhitespaceTokenizer wt = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);
ICharTermAttribute term = cgf.AddAttribute<ICharTermAttribute>();
cgf.Reset();
assertTrue(cgf.IncrementToken());
assertEquals("How", term.ToString());
assertTrue(cgf.IncrementToken());
assertEquals("How_the", term.ToString());
assertTrue(cgf.IncrementToken());
assertEquals("the", term.ToString());
assertTrue(cgf.IncrementToken());
assertEquals("the_s", term.ToString());
cgf.Dispose();
wt.Reader = new StringReader(input);
cgf.Reset();
assertTrue(cgf.IncrementToken());
assertEquals("How", term.ToString());
}
