public override Token Next()
{
var code = _reader.Read();
if (code.IsNull())
{
return(null);
}
if (code.IsLetterCase() || code.IsNumeralCase())
{
_reader.Seek(_reader.Position - 1);
return(base.Next());
}
else if (code.IsCjkCase())
{
var nextCode = _reader.Read();
if (nextCode.IsNull())
{
if (_beginState)
{
return(new Token(code.ToString(), TokenType.CJK));
}
return(null);
}
if (nextCode.IsCjkCase())
{
_beginState = false;
if (_reader.Peek().IsCjkCase())
{
_reader.Seek(_reader.Position - 1);
}
return(new Token(new string(new char[] { code, nextCode }), TokenType.CJK));
}
//may be code is a one of letter&numeral&punc.
_reader.Seek(_reader.Position - 2);
return(base.Next());
}
_beginState = true;
return(new Token(code.ToString(), TokenType.PUNC));
}