Core.DlrEngine.GetTokenInfos C# (CSharp) Method

        public virtual List<TokenInfo> GetTokenInfos(string code)
        {
            var tokenizer = new Tokenizer(true);
            tokenizer.Initialize(CreateSourceUnit(code));
            return new List<TokenInfo>(tokenizer.ReadTokens(code.Length));
        }

 public static List<Run> Colorize(DlrEngine engine, string code, Action<Run, TokenInfo> proc)
 {
     var result = new List<Run>();
     int position = 0;
     foreach (TokenInfo token in engine.GetTokenInfos(code)) {
         result.Add(CreateLeadingWhitespaceRun(code, position, token));
         var run = CreateTextRun(code, token);
         if (proc != null)
             proc(run, token);
         result.Add(run);
         position = token.SourceSpan.Start.Index + token.SourceSpan.Length;
     }
     return result;
 }