private static void ParseTokens(string text, TokenType type)
{
Lexer lexer = new Lexer(text);
string[] values = text.Split(' ');
Token token;
foreach (string value in values)
{
token = lexer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual(type, token.TokenType);
Assert.AreEqual(value, token.Value);
}
Assert.IsNull(lexer.NextToken());
}