BalticAmadeus.FluentMdx.Lexer.Lexer.Tokenize C# (CSharp) Method

Lexer Class Documentation Show file Open project: BalticAmadeus/FluentMdx

Tokenize() public method

Transforms text into list of Token objects based on rules.

public Tokenize ( string source ) : IEnumerable
source	string	Text to tokenize.
return	IEnumerable

        public IEnumerable<Token> Tokenize(string source)
        {
            int currentIndex = 0;

            while (currentIndex < source.Length)
            {
                if (source[currentIndex] == ' ')
                {
                    currentIndex++;
                    continue;
                }

                TokenDefinition matchedDefinition = null;
                int matchLength = 0;

                foreach (var rule in _tokenDefinitions)
                {
                    var match = rule.Pattern.Match(source, currentIndex);

                    if (!match.Success || (match.Index - currentIndex) != 0)
                        continue;

                    matchedDefinition = rule;
                    matchLength = match.Length;
                    break;
                }

                if (matchedDefinition == null)
                    throw new Exception(string.Format("Unrecognized symbol '{0}'.", source[currentIndex]));

                var value = source.Substring(currentIndex, matchLength);

                if (!matchedDefinition.IsIgnored)
                    yield return new Token(matchedDefinition.Type, value);

                currentIndex += matchLength;
            }

            yield return new Token(TokenType.LastToken, string.Empty);
        }

Lexer

Tokenize