ShaderTools.Hlsl.Parser.HlslLexer.Lex C# (CSharp) Method

        public SyntaxToken Lex(LexerMode mode)
        {
            // First check if we're in the middle of expanding a macro reference token.
            if (_expandedMacroTokens != null)
            {
                var result = _expandedMacroTokens[_expandedMacroIndex++];
                if (_expandedMacroIndex == _expandedMacroTokens.Count)
                    _expandedMacroTokens = null;
                return result;
            }

            _mode = mode;

            SyntaxToken token;
            switch (_mode)
            {
                case LexerMode.Syntax:
                    token = LexSyntaxToken();
                    break;
                case LexerMode.Directive:
                    token = LexDirectiveToken();
                    break;
                default:
                    throw new ArgumentOutOfRangeException();
            }

            // Swallow end-of-file tokens from include files.
            if (token.Kind == SyntaxKind.EndOfFileToken && _includeStack.Count > 1)
            {
                var originalToken = token;

                PopIncludeContext();
                token = Lex(mode);
                token = token.WithLeadingTrivia(originalToken.LeadingTrivia.AddRange(token.LeadingTrivia));

                // this is a bit weird, but we need to also update the leading trivia on the macro reference,
                // because that's what we use when outputting code.
                if (token.MacroReference != null)
                    token = token.WithOriginalMacroReference(token.MacroReference.WithLeadingTrivia(token.LeadingTrivia), token.IsFirstTokenInMacroExpansion);
            }

            // Expand macros and attach as a special kind of trivia.
            if (token.Kind == SyntaxKind.IdentifierToken && ExpandMacros)
            {
                List<SyntaxToken> expandedTokens;
                if (TryExpandMacro(token, new BaseMacroExpansionLexer(this), out expandedTokens))
                {
                    if (expandedTokens.Count == 0) // Can happen for macros with empty body.
                    {
                        // Attach macro call as leading trivia on next token.
                        var originalToken = token;
                        token = Lex(mode);

                        var leadingTrivia = new List<SyntaxNode>();
                        leadingTrivia.AddRange(originalToken.LeadingTrivia);
                        leadingTrivia.Add(new SyntaxTrivia(SyntaxKind.EmptyExpandedMacroTrivia, originalToken.Text, originalToken.SourceRange, originalToken.Span, ImmutableArray<Diagnostic>.Empty));
                        leadingTrivia.AddRange(originalToken.TrailingTrivia);
                        leadingTrivia.AddRange(token.LeadingTrivia);
                        token = token.WithLeadingTrivia(leadingTrivia.ToImmutableArray());
                    }
                    else
                    {
                        if (expandedTokens.Count > 1)
                        {
                            _expandedMacroTokens = expandedTokens;
                            _expandedMacroIndex = 1;
                        }
                        token = expandedTokens[0];
                    }
                }
            }

            return token;
        }

        public static IReadOnlyList<SyntaxToken> ParseAllTokens(SourceText sourceText, IIncludeFileSystem fileSystem = null)
        {
            var tokens = new List<SyntaxToken>();

            var lexer = new HlslLexer(sourceText, fileSystem: fileSystem);
            SyntaxToken token;
            do
            {
                tokens.Add(token = lexer.Lex(LexerMode.Syntax));
            } while (token.Kind != SyntaxKind.EndOfFileToken);

            return tokens;
        }