internal int GetTokenCount()
{
return m_countTokens;
}
internal void GetTokens(TokenizerStream stream, int maxNum, bool endAfterKet)
{
while (maxNum == -1 || stream.GetTokenCount() < maxNum)
{
int i = -1;
byte ch;
int cb = 0;
bool inLiteral = false;
bool inQuotedString = false;
StringMaker m = _maker;
m._outStringBuilder = null;
m._outIndex = 0;
BEGINNING:
if (_inSavedCharacter != -1)
{
i = _inSavedCharacter;
_inSavedCharacter = -1;
}
else
{
switch (_inTokenSource)
{
case TokenSource.UnicodeByteArray:
if (_inIndex + 1 >= _inSize)
{
stream.AddToken(-1);
return;
}
i = (int)((_inBytes[_inIndex + 1] << 8) + _inBytes[_inIndex]);
_inIndex += 2;
break;
case TokenSource.UTF8ByteArray:
if (_inIndex >= _inSize)
{
stream.AddToken(-1);
return;
}
i = (int)(_inBytes[_inIndex++]);
// single byte -- case, early out as we're done already
if ((i & 0x80) == 0x00)
{
break;
}
// to decode the lead byte switch on the high nibble
// shifted down so the switch gets dense integers
switch ((i & 0xf0) >> 4)
{
case 0x8: // 1000 (together these 4 make 10xxxxx)
case 0x9: // 1001
case 0xa: // 1010
case 0xb: // 1011
// trail byte is an error
throw new XmlSyntaxException(LineNo);
case 0xc: // 1100 (these two make 110xxxxx)
case 0xd: // 1101
// two byte encoding (1 trail byte)
i &= 0x1f;
cb = 2;
break;
case 0xe: // 1110 (this gets us 1110xxxx)
// three byte encoding (2 trail bytes)
i &= 0x0f;
cb = 3;
break;
case 0xf: // 1111 (and finally 1111xxxx)
// 4 byte encoding is an error
throw new XmlSyntaxException(LineNo);
}
// at least one trail byte, fetch it
if (_inIndex >= _inSize)
{
throw new XmlSyntaxException(LineNo, Environment.GetResourceString("XMLSyntax_UnexpectedEndOfFile"));
}
ch = _inBytes[_inIndex++];
// must be trail byte encoding
if ((ch & 0xc0) != 0x80)
{
throw new XmlSyntaxException(LineNo);
}
i = (i << 6) | (ch & 0x3f);
// done now if 2 byte encoding, otherwise go for 3
if (cb == 2)
{
break;
}
if (_inIndex >= _inSize)
{
throw new XmlSyntaxException(LineNo, Environment.GetResourceString("XMLSyntax_UnexpectedEndOfFile"));
}
ch = _inBytes[_inIndex++];
// must be trail byte encoding
if ((ch & 0xc0) != 0x80)
{
throw new XmlSyntaxException(LineNo);
}
i = (i << 6) | (ch & 0x3f);
break;
case TokenSource.ASCIIByteArray:
if (_inIndex >= _inSize)
{
stream.AddToken(-1);
return;
}
i = (int)(_inBytes[_inIndex++]);
break;
case TokenSource.CharArray:
if (_inIndex >= _inSize)
{
stream.AddToken(-1);
return;
}
i = (int)(_inChars[_inIndex++]);
break;
case TokenSource.String:
if (_inIndex >= _inSize)
{
stream.AddToken(-1);
return;
}
i = (int)(_inString[_inIndex++]);
break;
case TokenSource.NestedStrings:
if (_inNestedSize != 0)
{
if (_inNestedIndex < _inNestedSize)
{
i = _inNestedString[_inNestedIndex++];
break;
}
_inNestedSize = 0;
}
if (_inIndex >= _inSize)
{
stream.AddToken(-1);
return;
}
i = (int)(_inString[_inIndex++]);
if (i != '{')
{
break;
}
for (int istr = 0; istr < _searchStrings.Length; istr++)
{
if (0 == String.Compare(_searchStrings[istr], 0, _inString, _inIndex - 1, _searchStrings[istr].Length, StringComparison.Ordinal))
{
_inNestedString = _replaceStrings[istr];
_inNestedSize = _inNestedString.Length;
_inNestedIndex = 1;
i = _inNestedString[0];
_inIndex += _searchStrings[istr].Length - 1;
break;
}
}
break;
default:
i = _inTokenReader.Read();
if (i == -1)
{
stream.AddToken(-1);
return;
}
break;
}
}
if (!inLiteral)
{
switch (i)
{
// skip whitespace
case intSpace:
case intTab:
case intCR:
goto BEGINNING;
// count linefeeds
case intLF:
LineNo++;
goto BEGINNING;
case intOpenBracket:
_inProcessingTag++;
stream.AddToken(bra);
continue;
case intCloseBracket:
_inProcessingTag--;
stream.AddToken(ket);
if (endAfterKet)
{
return;
}
continue;
case intEquals:
stream.AddToken(equals);
continue;
case intSlash:
if (_inProcessingTag != 0)
{
stream.AddToken(slash);
continue;
}
break;
case intQuest:
if (_inProcessingTag != 0)
{
stream.AddToken(quest);
continue;
}
break;
case intBang:
if (_inProcessingTag != 0)
{
stream.AddToken(bang);
continue;
}
break;
case intDash:
if (_inProcessingTag != 0)
{
stream.AddToken(dash);
continue;
}
break;
case intQuote:
inLiteral = true;
inQuotedString = true;
goto BEGINNING;
}
}
else
{
switch (i)
{
case intOpenBracket:
if (!inQuotedString)
{
_inSavedCharacter = i;
stream.AddToken(cstr);
stream.AddString(this.GetStringToken());
continue;
}
break;
case intCloseBracket:
case intEquals:
case intSlash:
if (!inQuotedString && _inProcessingTag != 0)
{
_inSavedCharacter = i;
stream.AddToken(cstr);
stream.AddString(this.GetStringToken());
continue;
}
break;
case intQuote:
if (inQuotedString)
{
stream.AddToken(cstr);
stream.AddString(this.GetStringToken());
continue;
}
break;
case intTab:
case intCR:
case intSpace:
if (!inQuotedString)
{
stream.AddToken(cstr);
stream.AddString(this.GetStringToken());
continue;
}
break;
// count linefeeds
case intLF:
LineNo++;
if (!inQuotedString)
{
stream.AddToken(cstr);
stream.AddString(this.GetStringToken());
continue;
}
break;
}
}
inLiteral = true;
// add character to the string
if (m._outIndex < StringMaker.outMaxSize)
{
// easy case
m._outChars[m._outIndex++] = (char)i;
}
else
{
if (m._outStringBuilder == null)
{
// OK, first check if we have to init the StringBuilder
m._outStringBuilder = new StringBuilder();
}
// OK, copy from _outChars to _outStringBuilder
m._outStringBuilder.Append(m._outChars, 0, StringMaker.outMaxSize);
// reset _outChars pointer
m._outChars[0] = (char)i;
m._outIndex = 1;
}
goto BEGINNING;
}
}
