private TokenStream RewriteVariables(TokenStream tokens)
{
List<Token> result = new List<Token>();
foreach (Token token in tokens)
{
if (token.Type == TokenType.Word)
{
// Create a variable token.
result.Add(new Token(TokenType.Variable, (string)token.Value));
}
else
{
result.Add(token);
}
}
return new TokenStream(result, tokens.Notation);
}