public IEnumerable<Token> Lex()
{
IEnumerable<Token> tokens = null;
while ((tokens = LexLine()) != null)
{
foreach (var token in tokens)
{
yield return token;
}
}
for (int i = 0; i < eater.OpenIndents; ++i)
{
yield return new Token() { Kind = TokenKind.EndGroup, Line = lineNum };
}
yield return new Token() { Kind = TokenKind.EOF, Line = lineNum };
}
public void ItLexesMultpleCloseEndGroups()
{
var lexer = new Lexer(
@"a
b
c
d".ToStream());
var stream = lexer.Lex().ToArray();
AssertTokenStreamEquals(stream, new []
{
new Token() { Kind = TokenKind.Statement, Value = "a"},
new Token() { Kind = TokenKind.EOL},
new Token() { Kind = TokenKind.StartGroup },
new Token() { Kind = TokenKind.Statement, Value = "b"},
new Token() { Kind = TokenKind.EOL},
new Token() { Kind = TokenKind.StartGroup },
new Token() { Kind = TokenKind.Statement, Value = "c"},
new Token() { Kind = TokenKind.EOL},
new Token() { Kind = TokenKind.EndGroup },
new Token() { Kind = TokenKind.EndGroup },
new Token() { Kind = TokenKind.Statement, Value = "d"},
new Token() { Kind = TokenKind.EOL},
new Token() { Kind = TokenKind.EOF},
});
}