Axiom.Scripting.Compiler.Parser.ScriptLexer.SetToken C# (CSharp) Method

		private void SetToken( StringBuilder lexeme, uint line, String source, List<ScriptToken> tokens )
		{
			const char newline = '\n', openBrace = '{', closeBrace = '}', colon = ':', quote = '\"', var = '$';

			ScriptToken token = new ScriptToken();
			token.lexeme = lexeme.ToString();
			token.line = line;
			token.file = source;
			bool ignore = false;

			// Check the user token map first
			if ( lexeme.Length == 1 && lexeme[ 0 ] == newline )
			{
				token.type = Tokens.Newline;
				if ( tokens.Count != 0 && tokens[ tokens.Count - 1 ].type == Tokens.Newline )
					ignore = true;
			}
			else if ( lexeme.Length == 1 && lexeme[ 0 ] == openBrace )
				token.type = Tokens.LeftBrace;
			else if ( lexeme.Length == 1 && lexeme[ 0 ] == closeBrace )
				token.type = Tokens.RightBrace;
			else if ( lexeme.Length == 1 && lexeme[ 0 ] == colon )
				token.type = Tokens.Colon;
			else if ( lexeme[ 0 ] == var )
				token.type = Tokens.Variable;
			else
			{
				// This is either a non-zero length phrase or quoted phrase
				if ( lexeme.Length >= 2 && lexeme[ 0 ] == quote && lexeme[ lexeme.Length - 1 ] == quote )
				{
					token.type = Tokens.Quote;
				}
				else
				{
					token.type = Tokens.Word;
				}
			}

			if ( !ignore )
				tokens.Add( token );
		}