private void SetToken( StringBuilder lexeme, uint line, String source, List<ScriptToken> tokens )
{
const char newline = '\n', openBrace = '{', closeBrace = '}', colon = ':', quote = '\"', var = '$';
ScriptToken token = new ScriptToken();
token.lexeme = lexeme.ToString();
token.line = line;
token.file = source;
bool ignore = false;
// Check the user token map first
if ( lexeme.Length == 1 && lexeme[ 0 ] == newline )
{
token.type = Tokens.Newline;
if ( tokens.Count != 0 && tokens[ tokens.Count - 1 ].type == Tokens.Newline )
ignore = true;
}
else if ( lexeme.Length == 1 && lexeme[ 0 ] == openBrace )
token.type = Tokens.LeftBrace;
else if ( lexeme.Length == 1 && lexeme[ 0 ] == closeBrace )
token.type = Tokens.RightBrace;
else if ( lexeme.Length == 1 && lexeme[ 0 ] == colon )
token.type = Tokens.Colon;
else if ( lexeme[ 0 ] == var )
token.type = Tokens.Variable;
else
{
// This is either a non-zero length phrase or quoted phrase
if ( lexeme.Length >= 2 && lexeme[ 0 ] == quote && lexeme[ lexeme.Length - 1 ] == quote )
{
token.type = Tokens.Quote;
}
else
{
token.type = Tokens.Word;
}
}
if ( !ignore )
tokens.Add( token );
}