internal virtual void FillTokens()
{
StringBuilder sb = new StringBuilder();
char[] buffer = new char[256];
while (true)
{
int count = input.Read(buffer, 0, buffer.Length);
//.NET TextReader.Read(buff, int, int) returns 0, not -1 on no chars
// but in some cases, such as MockCharFilter, it overloads read and returns -1
// so we should handle both 0 and -1 values
if (count <= 0)
{
break;
}
sb.Append(buffer, 0, count);
//System.out.println("got count=" + count);
}
//System.out.println("fillTokens: " + sb);
InputLength = sb.Length;
string[] parts = sb.ToString().Split(' ');
Tokens = new List<Token>();
int pos = 0;
int maxPos = -1;
int offset = 0;
//System.out.println("again");
foreach (string part in parts)
{
string[] overlapped = part.Split('/');
bool firstAtPos = true;
int minPosLength = int.MaxValue;
foreach (string part2 in overlapped)
{
int colonIndex = part2.IndexOf(':');
string token;
int posLength;
if (colonIndex != -1)
{
token = part2.Substring(0, colonIndex);
posLength = Convert.ToInt32(part2.Substring(1 + colonIndex));
}
else
{
token = part2;
posLength = 1;
}
maxPos = Math.Max(maxPos, pos + posLength);
minPosLength = Math.Min(minPosLength, posLength);
Token t = new Token(token, offset, offset + 2 * posLength - 1);
t.PositionLength = posLength;
t.PositionIncrement = firstAtPos ? 1 : 0;
firstAtPos = false;
//System.out.println(" add token=" + t + " startOff=" + t.StartOffset() + " endOff=" + t.EndOffset());
Tokens.Add(t);
}
pos += minPosLength;
offset = 2 * pos;
}
Debug.Assert(maxPos <= pos, "input string mal-formed: posLength>1 tokens hang over the end");
}
}