RTools.Util.StreamTokenizer.Tokenize C# (CSharp) Method

        public bool Tokenize(List<Token> tokens)
        {
            Token token;
            this.lineNumber = 1;

            while (NextToken(out token))
            {
                if (token == null) throw new NullReferenceException(
                        "StreamTokenizer: Tokenize: Got a null token from NextToken.");
                tokens.Add(token);
            }

            // add the last token returned (EOF)
            tokens.Add(token);
            return (true);
        }