public override sealed bool IncrementToken()
{
if (tokens.Count > 0)
{
Debug.Assert(current != null);
CompoundToken token = tokens.First.Value; tokens.RemoveFirst();
RestoreState(current); // keep all other attributes untouched
termAtt.SetEmpty().Append(token.txt);
offsetAtt.SetOffset(token.startOffset, token.endOffset);
posIncAtt.PositionIncrement = 0;
return true;
}
current = null; // not really needed, but for safety
if (input.IncrementToken())
{
// Only words longer than minWordSize get processed
if (termAtt.Length >= this.minWordSize)
{
Decompose();
// only capture the state if we really need it for producing new tokens
if (tokens.Count > 0)
{
current = CaptureState();
}
}
// return original token:
return true;
}
else
{
return false;
}
}
