public List<Token> process(TextReader pTextReader)
{
D.isNull(pTextReader);
_tokens = new List<Token>();
_textReader = pTextReader;
_endOfFile = false;
readNextChar();
_currentLine = 1;
_currentPosition = 0;
_currentTokenStartPosition = 0;
Token t;
do {
t = readNextToken();
t.LineNr = _currentLine;
t.LinePosition = _currentTokenStartPosition;
_currentTokenStartPosition = _currentPosition;
_tokens.Add(t);
#if WRITE_DEBUG
Console.WriteLine(t.LineNr + ": " + t.getTokenType().ToString() + " " + t.getTokenString());
#endif
} while(t.getTokenType() != Token.TokenType.EOF);
_textReader.Close();
_textReader.Dispose();
return _tokens;
}