public override bool incrementToken()
{
clearAttributes();
if (delimitersCount == -1)
{
int length = 0;
delimiterPositions.Add(0);
while (true)
{
int c = input.read();
if (c < 0)
{
break;
}
length++;
if (c == delimiter)
{
delimiterPositions.Add(length);
resultToken.Append(replacement);
}
else
{
resultToken.Append((char)c);
}
}
delimitersCount = delimiterPositions.Count;
if (delimiterPositions[delimitersCount - 1] < length)
{
delimiterPositions.Add(length);
delimitersCount++;
}
if (resultTokenBuffer.Length < resultToken.Length)
{
resultTokenBuffer = new char[resultToken.Length];
}
resultToken.getChars(0, resultToken.Length, resultTokenBuffer, 0);
resultToken.Length = 0;
int idx = delimitersCount - 1 - skip;
if (idx >= 0)
{
// otherwise its ok, because we will skip and return false
endPosition = delimiterPositions[idx];
}
finalOffset = correctOffset(length);
posAtt.PositionIncrement = 1;
}
else
{
posAtt.PositionIncrement = 0;
}
while (skipped < delimitersCount - skip - 1)
{
int start = delimiterPositions[skipped];
termAtt.copyBuffer(resultTokenBuffer, start, endPosition - start);
offsetAtt.setOffset(correctOffset(start), correctOffset(endPosition));
skipped++;
return true;
}
return false;
}