public override bool IncrementToken()
{
ClearAttributes();
if (!started)
{
started = true;
gramSize = minGram;
char[] chars = new char[1024];
inStr = input.ReadToEnd(); // remove any trailing empty strings
inLen = inStr.Length;
}
if (pos + gramSize > inLen)
{ // if we hit the end of the string
pos = 0; // reset to beginning of string
gramSize++; // increase n-gram size
if (gramSize > maxGram) // we are done
return false;
if (pos + gramSize > inLen)
return false;
}
int oldPos = pos;
pos++;
termAtt.SetTermBuffer(inStr, oldPos, gramSize);
offsetAtt.SetOffset(CorrectOffset(oldPos), CorrectOffset(oldPos + gramSize));
return true;
}