public override bool IncrementToken()
{
// get the next piece of input
if (savedState != null)
{
RestoreState(savedState);
savedState = null;
SaveTermBuffer();
return true;
}
else if (!input.IncrementToken())
{
return false;
}
/* We build n-grams before and after stopwords.
* When valid, the buffer always contains at least the separator.
* If its empty, there is nothing before this stopword.
*/
if (lastWasCommon || (Common && buffer.Length > 0))
{
savedState = CaptureState();
GramToken();
return true;
}
SaveTermBuffer();
return true;
}
public virtual void TestReset() { const string input = "How the s a brown s cow d like A B thing?"; WhitespaceTokenizer wt = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(input)); CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords); ICharTermAttribute term = cgf.AddAttribute<ICharTermAttribute>(); cgf.Reset(); assertTrue(cgf.IncrementToken()); assertEquals("How", term.ToString()); assertTrue(cgf.IncrementToken()); assertEquals("How_the", term.ToString()); assertTrue(cgf.IncrementToken()); assertEquals("the", term.ToString()); assertTrue(cgf.IncrementToken()); assertEquals("the_s", term.ToString()); cgf.Dispose(); wt.Reader = new StringReader(input); cgf.Reset(); assertTrue(cgf.IncrementToken()); assertEquals("How", term.ToString()); }