public void ProcessOneTokenWithSpacesAndComment()
{
Lexer tokenizer = new Lexer(" \"This is a token \" token \"This another comment\"");
Token token = tokenizer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual("token", token.Value);
Assert.AreEqual(TokenType.Name, token.Type);
token = tokenizer.NextToken();
Assert.IsNull(token);
}