Boo.BooLangStudioSpecs.AutoTokenTestFixture.BuildTokens C# (CSharp) Method

        public virtual void BuildTokens(string codeLine, int startOffset)
        {
            bool moreTokens = true;
              int state = 0;
              scanner.SetSource(codeLine, startOffset);
              while(moreTokens)
              {
            TokenInfo token = new TokenInfo();
            moreTokens = scanner.ScanTokenAndProvideInfoAboutIt(token, ref state);
            if (moreTokens)
              // no EOL token, since the IDE doesn't care about it anywho
              tokens.Add(token);
              }
        }