public void TokenizeTwoStrings()
{
var tokenizer = new Tokenizer("hello world");
var token = tokenizer.NextToken;
Assert.IsNotNull(token);
Assert.AreEqual(TokenKind.Value, token.Kind);
Assert.AreEqual("hello", token.Value);
Assert.AreEqual("world", tokenizer.RestOfInput);
token = tokenizer.NextToken;
Assert.IsNotNull(token);
Assert.AreEqual(TokenKind.Value, token.Kind);
Assert.AreEqual("world", token.Value);
Assert.AreEqual("", tokenizer.RestOfInput);
}