public Token[] Tokenize()
{
if (Text == null)
{
throw new ArgumentNullException("Text");
}
MakeImmutable();
var tokens = new List <Token>();
int index = 0;
while (index < Text.Length)
{
MatchResult match = tree.Match(Text, index);
if (match.Found)
{
string dummyText = Text.Substring(index, match.Index - index);
var dummyToken = new Token(dummyText, null);
tokens.Add(dummyToken);
var realToken = new Token(match.GetText(), match.Tags);
index = match.Index + match.Length;
tokens.Add(realToken);
}
else
{
string dummyText = Text.Substring(index);
var dummyToken = new Token(dummyText, null);
tokens.Add(dummyToken);
index = Text.Length;
}
}
return(tokens.ToArray());
}