public void ProcessTokenAndString()
{
Lexer tokenizer = new Lexer("token 'string'");
Token token;
token = tokenizer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual("token", token.Value);
Assert.AreEqual(TokenType.Name, token.Type);
token = tokenizer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual("string", token.Value);
Assert.AreEqual(TokenType.String, token.Type);
}