public Token NextToken()
{
if (this.tokens.Count > 0)
return this.tokens.Pop();
int ich = this.NextCharSkippingWhiteSpaces();
if (ich == -1)
return null;
char ch = (char)ich;
if (operators2.Any(op => op[0] == ch))
{
int ich2 = this.NextChar();
if (ich2 >= 0)
{
string op2 = ch.ToString() + ((char)ich2).ToString();
int ich3 = this.NextChar();
if (ich3 >= 0)
{
string op3 = op2 + ((char)ich3).ToString();
if (operators3.Contains(op3))
return new Token(op3, TokenType.Operator);
this.PushChar(ich3);
}
if (operators2.Contains(op2))
return new Token(op2, TokenType.Operator);
this.PushChar(ich2);
}
}
if (separators2.Any(op => op[0] == ch))
{
int ich2 = this.NextChar();
if (ich2 >= 0)
{
string sep2 = ch.ToString() + ((char)ich2).ToString();
if (separators2.Contains(sep2))
return new Token(sep2, TokenType.Separator);
this.PushChar(ich2);
}
}
if (operators.Contains(ch))
return new Token(ch.ToString(), TokenType.Operator);
if (separators.Contains(ch))
return new Token(ch.ToString(), TokenType.Separator);
if (char.IsDigit(ch))
return this.NextInteger(ch);
if (ch == '"')
return this.NextString();
if (IsNameChar(ch))
return this.NextName(ch);
throw new ParserException(string.Format("Unexpected '{0}'", ch));
}
public void GetArithmeticOperators() { Lexer lexer = new Lexer("+-*/"); Token token = lexer.NextToken(); Assert.IsNotNull(token); Assert.AreEqual(TokenType.Operator, token.Type); Assert.AreEqual("+", token.Value); token = lexer.NextToken(); Assert.IsNotNull(token); Assert.AreEqual(TokenType.Operator, token.Type); Assert.AreEqual("-", token.Value); token = lexer.NextToken(); Assert.IsNotNull(token); Assert.AreEqual(TokenType.Operator, token.Type); Assert.AreEqual("*", token.Value); token = lexer.NextToken(); Assert.IsNotNull(token); Assert.AreEqual(TokenType.Operator, token.Type); Assert.AreEqual("/", token.Value); Assert.IsNull(lexer.NextToken()); }