public override bool IncrementToken()
{
while (true)
{
if (!hasSavedState)
{
// process a new input word
if (!input.IncrementToken())
{
return false;
}
int termLength = termAttribute.Length;
char[] termBuffer = termAttribute.Buffer();
accumPosInc += posIncAttribute.PositionIncrement;
iterator.SetText(termBuffer, termLength);
iterator.Next();
// word of no delimiters, or protected word: just return it
if ((iterator.current == 0 && iterator.end == termLength) || (protWords != null && protWords.Contains(termBuffer, 0, termLength)))
{
posIncAttribute.PositionIncrement = accumPosInc;
accumPosInc = 0;
return true;
}
// word of simply delimiters
if (iterator.end == WordDelimiterIterator.DONE && !Has(PRESERVE_ORIGINAL))
{
// if the posInc is 1, simply ignore it in the accumulation
if (posIncAttribute.PositionIncrement == 1)
{
accumPosInc--;
}
continue;
}
SaveState();
hasOutputToken = false;
hasOutputFollowingOriginal = !Has(PRESERVE_ORIGINAL);
lastConcatCount = 0;
if (Has(PRESERVE_ORIGINAL))
{
posIncAttribute.PositionIncrement = accumPosInc;
accumPosInc = 0;
return true;
}
}
// at the end of the string, output any concatenations
if (iterator.end == WordDelimiterIterator.DONE)
{
if (!concat.Empty)
{
if (FlushConcatenation(concat))
{
return true;
}
}
if (!concatAll.Empty)
{
// only if we haven't output this same combo above!
if (concatAll.subwordCount > lastConcatCount)
{
concatAll.WriteAndClear();
return true;
}
concatAll.Clear();
}
// no saved concatenations, on to the next input word
hasSavedState = false;
continue;
}
// word surrounded by delimiters: always output
if (iterator.SingleWord)
{
GeneratePart(true);
iterator.Next();
return true;
}
int wordType = iterator.Type;
// do we already have queued up incompatible concatenations?
if (!concat.Empty && (concat.type & wordType) == 0)
{
if (FlushConcatenation(concat))
{
hasOutputToken = false;
return true;
}
hasOutputToken = false;
}
// add subwords depending upon options
if (ShouldConcatenate(wordType))
{
if (concat.Empty)
{
concat.type = wordType;
}
Concatenate(concat);
}
// add all subwords (catenateAll)
if (Has(CATENATE_ALL))
{
Concatenate(concatAll);
}
// if we should output the word or number part
if (ShouldGenerateParts(wordType))
{
GeneratePart(false);
iterator.Next();
return true;
}
iterator.Next();
}
}