public Tokenize ( string expression ) : IEnumerable |
||
expression | string | |
return | IEnumerable |
public IEnumerable<IToken> Tokenize(string expression)
{
_states = new Stack<IState>();
_currentState = new LiteralState();
_characters = expression.GetEnumerator();
while (_characters.MoveNext())
{
var token = _currentState.Handle(this);
if (token != null)
{
yield return token;
}
}
}
public void EscapeStateTokenizeNumeric() { var lexer = new Lexer(); var tokens = lexer.Tokenize("\\d"); var expected = new NumericToken(); Assert.AreEqual(tokens.Last(), expected); }