Tokenizer.ReadToken C# (CSharp) Method

    public Token ReadToken() 
    {
      _token = ReadTokenInternal();
      return _token;
    }

    private void OnValidate()
    {
        System.Text.StringBuilder sb;
        // ------------------------------------------------------------------
        // Just tokenized it
        // ------------------------------------------------------------------
        sb    = new System.Text.StringBuilder();
        lexer = new Tokenizer(new StringReader(testString), "TokenizerTest");
        var token = lexer.ReadToken();

        while (token != null)
        {
            sb.Append(Inspector.Inspect(token) + " ");
            token = lexer.ReadToken();
        }
        tokensString = sb.ToString();

        // ------------------------------------------------------------------
        // Parse scheme
        // ------------------------------------------------------------------
        lexer = new Tokenizer(new StringReader(testString), "TokenizerTest");
        sb    = new System.Text.StringBuilder();
        do
        {
            SObject result = Parser.Parse(lexer);
            if (result == null)
            {
                break;
            }
            sb.AppendLine(result.Inspect());
        } while (lexer.LastToken != null);
        resultString = sb.ToString();
    }