public void ProcessPunctuations()
{
string punct = "().|[];";
Lexer tokenizer = new Lexer(punct);
Token token;
for (int k = 0; k < punct.Length; k++)
{
token = tokenizer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual(punct[k], token.Value[0]);
Assert.AreEqual(1, token.Value.Length);
Assert.AreEqual(TokenType.Punctuation, token.Type);
}
}