CSharpUtils.Templates.Tokenizers.ExpressionTokenizer.Tokenize C# (CSharp) Метод

		public static void Tokenize(List<TemplateToken> Tokens, TokenizerStringReader StringReader)
		{
			//StringReader.SkipSpaces();
			while (StringReader.Available > 0)
			{
				switch (StringReader.Peek(2))
				{
					case "%}":
					case "}}":
						return;
				}

				var CharBase = StringReader.PeekChar();

				switch (CharBase)
				{
					// Ignore Spaces
					case ' ':
					case '\t':
					case '\r':
					case '\n':
					case '\v':
						StringReader.Skip();
						break;
					// String
					case '\'':
					case '"':
						{
							StringReader.SegmentSetStart();
							StringReader.Skip();
							while (true)
							{
								var Char = StringReader.ReadChar();
								if (Char == '\\')
								{
									StringReader.ReadChar();
								}
								else if (Char == CharBase)
								{
									break;
								}
							}
							StringReader.SegmentSetEnd();
							Tokens.Add(new StringLiteralTemplateToken()
							{
								Text = StringReader.SegmentGetSlice(),
							});
						}
						break;
					default:
						// Numbers
						if (IsDecimalDigit(CharBase))
						{
							StringReader.SegmentSetStart();
							StringReader.Skip();
							while (true)
							{
								var Char = StringReader.PeekChar();
								if (IsDecimalDigit(Char))
								{
									StringReader.Skip();
								}
								else
								{
									break;
								}
							}
							StringReader.SegmentSetEnd();
							Tokens.Add(new NumericLiteralTemplateToken()
							{
								Text = StringReader.SegmentGetSlice(),
							});
						}
						else
						{
							// Operators
							if (Operators2.Contains(StringReader.Peek(2)))
							{
								Tokens.Add(new OperatorTemplateToken()
								{
									Text = StringReader.ReadString(2),
								});
								break;
							}

							if (Operators1.Contains(CharBase))
							{
								Tokens.Add(new OperatorTemplateToken()
								{
									Text = StringReader.ReadChar().ToString(),
								});
								break;
							}

							if (IsAlpha(CharBase))
							{
								StringReader.SegmentSetStart();
								StringReader.Skip();
								while (true)
								{
									var Char = StringReader.PeekChar();
									if (IsAlphaNumeric(Char))
									{
										StringReader.Skip();
									}
									else
									{
										break;
									}
								}
								StringReader.SegmentSetEnd();
								Tokens.Add(new IdentifierTemplateToken()
								{
									Text = StringReader.SegmentGetSlice(),
								});
								break;
							}

							StringReader.Skip();

							throw (new Exception("Unknown Token Type : '" + CharBase + "'"));
						}
						break;
				}
			}
		}
	}

        public static List <TemplateToken> Tokenize(List <TemplateToken> Tokens, TokenizerStringReader StringReader)
        {
            while (true)
            {
                var Match = StringReader.Match(StartTagRegex);

                if (Match.Success)
                {
                    var RawText = StringReader.ReadString(Match.Index);
                    if (RawText.Length > 0)
                    {
                        Tokens.Add(new RawTemplateToken()
                        {
                            Text = RawText
                        });
                    }
                    var OpenTagTokenString = StringReader.ReadString(2);

                    // It is a comment.
                    if (OpenTagTokenString == "{#")
                    {
                        int Index = StringReader.IndexOf("#}");
                        if (Index == -1)
                        {
                            throw(new Exception("Not closing comment"));
                        }
                        StringReader.ReadString(Index + 2);
                    }
                    // It is either a expression tag or a special tag.
                    else
                    {
                        Tokens.Add(new OpenTagTemplateToken()
                        {
                            Text = OpenTagTokenString
                        });
                        {
                            ExpressionTokenizer.Tokenize(Tokens, StringReader);
                        }
                        var CloseTagTokenString         = StringReader.ReadString(2);
                        var ExpectedCloseTagTokenString = ((OpenTagTokenString == "{{") ? "}}" : "%}");
                        if (CloseTagTokenString != ExpectedCloseTagTokenString)
                        {
                            throw (new Exception("Expected '" + ExpectedCloseTagTokenString + "' but got '" + CloseTagTokenString + "'"));
                        }
                        Tokens.Add(new CloseTagTemplateToken()
                        {
                            Text = CloseTagTokenString
                        });
                        continue;
                    }
                }
                else
                {
                    var RawText = StringReader.ReadString();
                    if (RawText.Length > 0)
                    {
                        Tokens.Add(new RawTemplateToken()
                        {
                            Text = RawText
                        });
                    }
                    return(Tokens);
                }
            }
        }