public Token NextToken()
{
if (this.tokenstack.Count > 0)
{
return this.tokenstack.Pop();
}
int ch;
ch = this.NextCharSkipBlanksAndComments();
if (ch < 0)
return null;
if (char.IsLetter((char)ch) || ch == '_')
{
return this.NextName((char)ch);
}
if (char.IsDigit((char)ch))
{
return this.NextInteger((char)ch);
}
if (ch == StringDelimiter)
{
return this.NextString();
}
if (ch == SpecialCharMark)
{
int ch2 = this.NextChar();
return new Token() { Type = TokenType.Character, Value = ((char)ch2).ToString() };
}
if (ch == SymbolMark)
{
int ch2 = this.NextChar();
if (ch2 == '(')
return new Token() { Type = TokenType.Punctuation, Value = "#(" };
if (ch2 == '[')
return new Token() { Type = TokenType.Punctuation, Value = "#[" };
if (ch2 == '{')
return this.NextEnclosedSymbol();
if (ch2 == '\'')
return new Token() { Type = TokenType.Symbol, Value = this.NextString().Value };
this.PushChar(ch2);
return this.NextSymbol();
}
if (ch == ParameterMark)
{
int ch2 = this.PeekChar();
if (ch2 < 0 || !char.IsLetter((char)ch2))
{
return this.NextOperator((char)ch);
}
return this.NextParameter();
}
if (ch == SpecialDotNetTypeMark)
{
int ch2 = this.PeekChar();
if (ch2 < 0 || !char.IsLetter((char)ch2) || !char.IsUpper((char)ch2))
{
return this.NextOperator((char)ch);
}
return this.NextDotNetTypeName();
}
if (ch == SpecialDotNetInvokeMark)
{
return this.NextDotNetInvokeName();
}
if (Operators.IndexOf((char)ch) >= 0)
{
return this.NextOperator((char)ch);
}
if (Separators.IndexOf((char)ch) >= 0)
{
return this.NextPunctuation((char)ch);
}
throw new LexerException("Invalid Characater '" + (char)ch + "'");
}
public void ParseDotNetMethodMethodWithParenthesis() { Lexer tokenizer = new Lexer("(anObject !nativeMethod)"); Token token; token = tokenizer.NextToken(); Assert.IsNotNull(token); Assert.AreEqual("(", token.Value); Assert.AreEqual(TokenType.Punctuation, token.Type); token = tokenizer.NextToken(); Assert.IsNotNull(token); Assert.AreEqual("anObject", token.Value); Assert.AreEqual(TokenType.Name, token.Type); token = tokenizer.NextToken(); Assert.IsNotNull(token); Assert.AreEqual("!nativeMethod", token.Value); Assert.AreEqual(TokenType.Name, token.Type); token = tokenizer.NextToken(); Assert.IsNotNull(token); Assert.AreEqual(")", token.Value); Assert.AreEqual(TokenType.Punctuation, token.Type); token = tokenizer.NextToken(); Assert.IsNull(token); }