Lucene.Net.Analysis.MockTokenizer.IncrementToken C# (CSharp) Method

        public sealed override bool IncrementToken()
        {
            //Debug.Assert(!EnableChecks_Renamed || (StreamState == State.RESET || StreamState == State.INCREMENT), "IncrementToken() called while in wrong state: " + StreamState);
            ClearAttributes();
            for (; ; )
            {
                int startOffset;
                int cp;
                if (BufferedCodePoint >= 0)
                {
                    cp = BufferedCodePoint;
                    startOffset = BufferedOff;
                    BufferedCodePoint = -1;
                }
                else
                {
                    startOffset = Off;
                    cp = ReadCodePoint();
                }
                if (cp < 0)
                {
                    break;
                }
                else if (IsTokenChar(cp))
                {
                    int endOffset;
                    do
                    {
                        char[] chars = Character.ToChars(Normalize(cp));
                        for (int i = 0; i < chars.Length; i++)
                        {
                            TermAtt.Append(chars[i]);
                        }
                        endOffset = Off;
                        if (TermAtt.Length >= MaxTokenLength)
                        {
                            break;
                        }
                        cp = ReadCodePoint();
                    } while (cp >= 0 && IsTokenChar(cp));

                    if (TermAtt.Length < MaxTokenLength)
                    {
                        // buffer up, in case the "rejected" char can start a new word of its own
                        BufferedCodePoint = cp;
                        BufferedOff = endOffset;
                    }
                    else
                    {
                        // otherwise, its because we hit term limit.
                        BufferedCodePoint = -1;
                    }
                    int correctedStartOffset = CorrectOffset(startOffset);
                    int correctedEndOffset = CorrectOffset(endOffset);
                    Assert.True(correctedStartOffset >= 0);
                    Assert.True(correctedEndOffset >= 0);
                    Assert.True(correctedStartOffset >= LastOffset);
                    LastOffset = correctedStartOffset;
                    Assert.True(correctedEndOffset >= correctedStartOffset);
                    OffsetAtt.SetOffset(correctedStartOffset, correctedEndOffset);
                    if (state == -1 || RunAutomaton.IsAccept(state))
                    {
                        // either we hit a reject state (longest match), or end-of-text, but in an accept state
                        StreamState = State.INCREMENT;
                        return true;
                    }
                }
            }
            StreamState = State.INCREMENT_FALSE;
            return false;
        }