public sealed override bool IncrementToken()
{
//Debug.Assert(!EnableChecks_Renamed || (StreamState == State.RESET || StreamState == State.INCREMENT), "IncrementToken() called while in wrong state: " + StreamState);
ClearAttributes();
for (; ; )
{
int startOffset;
int cp;
if (BufferedCodePoint >= 0)
{
cp = BufferedCodePoint;
startOffset = BufferedOff;
BufferedCodePoint = -1;
}
else
{
startOffset = Off;
cp = ReadCodePoint();
}
if (cp < 0)
{
break;
}
else if (IsTokenChar(cp))
{
int endOffset;
do
{
char[] chars = Character.ToChars(Normalize(cp));
for (int i = 0; i < chars.Length; i++)
{
TermAtt.Append(chars[i]);
}
endOffset = Off;
if (TermAtt.Length >= MaxTokenLength)
{
break;
}
cp = ReadCodePoint();
} while (cp >= 0 && IsTokenChar(cp));
if (TermAtt.Length < MaxTokenLength)
{
// buffer up, in case the "rejected" char can start a new word of its own
BufferedCodePoint = cp;
BufferedOff = endOffset;
}
else
{
// otherwise, its because we hit term limit.
BufferedCodePoint = -1;
}
int correctedStartOffset = CorrectOffset(startOffset);
int correctedEndOffset = CorrectOffset(endOffset);
Assert.True(correctedStartOffset >= 0);
Assert.True(correctedEndOffset >= 0);
Assert.True(correctedStartOffset >= LastOffset);
LastOffset = correctedStartOffset;
Assert.True(correctedEndOffset >= correctedStartOffset);
OffsetAtt.SetOffset(correctedStartOffset, correctedEndOffset);
if (state == -1 || RunAutomaton.IsAccept(state))
{
// either we hit a reject state (longest match), or end-of-text, but in an accept state
StreamState = State.INCREMENT;
return true;
}
}
}
StreamState = State.INCREMENT_FALSE;
return false;
}