private void GramToken()
{
buffer.Append(termAttribute.Buffer(), 0, termAttribute.Length);
int endOffset = offsetAttribute.EndOffset();
ClearAttributes();
var length = buffer.Length;
var termText = termAttribute.Buffer();
if (length > termText.Length)
{
termText = termAttribute.ResizeBuffer(length);
}
buffer.CopyTo(0, termText, 0, length);
termAttribute.Length = length;
posIncAttribute.PositionIncrement = 0;
posLenAttribute.PositionLength = 2; // bigram
offsetAttribute.SetOffset(lastStartOffset, endOffset);
typeAttribute.Type = GRAM_TYPE;
buffer.Length = 0;
}
}