|
public Tokenize ( string expression ) : IEnumerable |
||
| expression | string | |
| return | IEnumerable |
|
public IEnumerable<IToken> Tokenize(string expression)
{
_states = new Stack<IState>();
_currentState = new LiteralState();
_characters = expression.GetEnumerator();
while (_characters.MoveNext())
{
var token = _currentState.Handle(this);
if (token != null)
{
yield return token;
}
}
}
public void EscapeStateTokenizeNumeric()
{
var lexer = new Lexer();
var tokens = lexer.Tokenize("\\d");
var expected = new NumericToken();
Assert.AreEqual(tokens.Last(), expected);
}
