public void TestTokenizerSimple()
{
Tokenizer tokenizer = new Tokenizer(new ExpressionDictionary(), "@book{ }");
List<AbstractToken> tokens = tokenizer.GetAllTokens().ToList();
Assert.AreEqual(4, tokens.Count());
Assert.AreEqual(new At("@"), tokens[0]);
Assert.AreEqual(new Text("book", 1), tokens[1]);
Assert.AreEqual(new OpeningBrace("{", 5), tokens[2]);
Assert.AreEqual(new ClosingBrace(" }" ,6), tokens[3]);
}