|
private static Tokenizer ( string input ) : IEnumerable |
||
| input | string | |
| return | IEnumerable |
|
private static IEnumerable<Token> Tokenizer (string input) {
foreach (var token in Tokenize) {
Match match;
while ((match = Regex.Match (input, token.Key)) != null && match.Success) {
yield return new Token { match = match, action = token.Value };
input = input.Remove (match.Index, match.Length);
}
}
}