VBF.Compilers.Scanners.Scanner.Read C# (CSharp) Method

        public Lexeme Read()
        {
            if (m_valuableCursor < m_valuableHistory.Count)
            {
                int fullCursor = m_valuableHistory[m_valuableCursor++];

                return m_fullHistory[fullCursor];
            }

            //m_triviaCache.Clear();

            while (true)
            {
                //run to next stopped state
                m_engine.Reset();
                m_lastTokenStart = m_source.PeekLocation();
                m_lastState = 0;

                m_lexemeValueBuilder.Clear();

                if (m_source.PeekChar() < 0)
                {
                    if (ThrowAtReadingAfterEndOfStream && HasReachedEndOfStream())
                    {
                        throw new ScannerException("The scanner has reached the end of stream.");
                    }
                    //return End Of Stream token
                    AddHistory(new Lexeme(m_scannerInfo, m_scannerInfo.EndOfStreamState,
                        new SourceSpan(m_lastTokenStart, m_lastTokenStart), null));

                    break;
                }
                while (true)
                {
                    int inputCharValue = m_source.PeekChar();

                    if (inputCharValue < 0)
                    {
                        //end of stream, treat as stopped
                        break;
                    }

                    char inputChar = (char)inputCharValue;
                    m_engine.Input(inputChar);

                    if (m_engine.IsAtStoppedState)
                    {
                        //stop immediately at a stopped state
                        break;
                    }
                    m_lastState = m_engine.CurrentState;

                    m_source.ReadChar();
                    m_lexemeValueBuilder.Append(inputChar);
                }

                //skip tokens that marked with "Skip" attribute
                bool isLastSkipped = IsLastTokenSkippable();

                if (isLastSkipped)
                {
                    AddHistory(new Lexeme(m_scannerInfo, m_lastState,
                        new SourceSpan(m_lastTokenStart, m_source.Location), m_lexemeValueBuilder.ToString()), false);
                }
                else
                {
                    AddHistory(new Lexeme(m_scannerInfo, m_lastState,
                        new SourceSpan(m_lastTokenStart, m_source.Location), m_lexemeValueBuilder.ToString()));

                    break;
                }
            }

            int lastTokenfullCursor = m_valuableHistory[m_valuableCursor - 1];
            return m_fullHistory[lastTokenfullCursor];
        }

        private Lexeme PeekLexeme(int lookAhead)
        {
            while (m_lookAheadQueue.Count < lookAhead)
            {
                //look ahead more
                m_lookAheadQueue.Enqueue(m_masterScanner.Read());
            }
            Lexeme lookAheadLexeme = m_lookAheadQueue[lookAhead - 1];

            return(lookAheadLexeme);
        }