public void BuildTokens(string line)
{
scanner.SetSource(line, 0);
bool moreTokens = true;
//TokenInfo token = new TokenInfo();
//int state = 0;
tokens.Clear();
while (moreTokens)
{
TokenInfo token = new TokenInfo();
moreTokens = scanner.ScanTokenAndProvideInfoAboutIt(token, ref _mlState);
if (moreTokens)
{
tokens.Add(token);
Console.WriteLine("type: " + token.Type.ToString() + " start: " + token.StartIndex.ToString() + " end: " + token.EndIndex + " ");
}
}
}
