public override Boolean IncrementToken() {
if (_buffer.Any()) {
var nextStem = _buffer.Dequeue();
RestoreState(_savedState);
_posIncAtt.SetPositionIncrement(0);
_termAtt.SetTermBuffer(nextStem.Stem, 0, nextStem.StemLength);
return true;
}
if (!input.IncrementToken())
return false;
var newTerms = _dedup
? _stemmer.UniqueStems(_termAtt.Term())
: _stemmer.Stem(_termAtt.Term());
foreach (var newTerm in newTerms)
_buffer.Enqueue(newTerm);
if (_buffer.Count == 0)
// we do not know this word, return it unchanged
return true;
var stem = _buffer.Dequeue();
_termAtt.SetTermBuffer(stem.Stem, 0, stem.StemLength);
if (_buffer.Count > 0)
_savedState = CaptureState();
return true;
}