|
private GetTokens ( |
||
| stream | ||
| maxNum | int | |
| endAfterKet | bool | |
| return | void | |
internal void GetTokens( TokenizerStream stream, int maxNum, bool endAfterKet )
{
while (maxNum == -1 || stream.GetTokenCount() < maxNum)
{
int i = -1;
byte ch;
int cb = 0;
bool inLiteral = false;
bool inQuotedString = false;
StringMaker m = _maker;
m._outStringBuilder = null;
m._outIndex = 0;
BEGINNING:
if (_inSavedCharacter != -1)
{
i = _inSavedCharacter;
_inSavedCharacter = -1;
}
else switch (_inTokenSource)
{
case TokenSource.UnicodeByteArray:
if (_inIndex + 1 >= _inSize)
{
stream.AddToken( -1 );
return;
}
i = (int)((_inBytes[_inIndex+1]<<8) + _inBytes[_inIndex]);
_inIndex += 2;
break;
case TokenSource.UTF8ByteArray:
if (_inIndex >= _inSize)
{
stream.AddToken( -1 );
return;
}
i = (int)(_inBytes[_inIndex++]);
// single byte -- case, early out as we're done already
if ((i & 0x80) == 0x00)
break;
// to decode the lead byte switch on the high nibble
// shifted down so the switch gets dense integers
switch ((i & 0xf0) >>4)
{
case 0x8: // 1000 (together these 4 make 10xxxxx)
case 0x9: // 1001
case 0xa: // 1010
case 0xb: // 1011
// trail byte is an error
throw new XmlSyntaxException( LineNo );
case 0xc: // 1100 (these two make 110xxxxx)
case 0xd: // 1101
// two byte encoding (1 trail byte)
i &= 0x1f;
cb = 2;
break;
case 0xe: // 1110 (this gets us 1110xxxx)
// three byte encoding (2 trail bytes)
i &= 0x0f;
cb = 3;
break;
case 0xf: // 1111 (and finally 1111xxxx)
// 4 byte encoding is an error
throw new XmlSyntaxException( LineNo );
}
// at least one trail byte, fetch it
if (_inIndex >= _inSize)
throw new XmlSyntaxException (LineNo, Environment.GetResourceString( "XMLSyntax_UnexpectedEndOfFile" ));
ch = _inBytes[_inIndex++];
// must be trail byte encoding
if ((ch & 0xc0) != 0x80)
throw new XmlSyntaxException( LineNo );
i = (i<<6) | (ch & 0x3f);
// done now if 2 byte encoding, otherwise go for 3
if (cb == 2)
break;
if (_inIndex >= _inSize)
throw new XmlSyntaxException (LineNo, Environment.GetResourceString( "XMLSyntax_UnexpectedEndOfFile" ));
ch = _inBytes[_inIndex++];
// must be trail byte encoding
if ((ch & 0xc0) != 0x80)
throw new XmlSyntaxException( LineNo );
i = (i<<6) | (ch & 0x3f);
break;
case TokenSource.ASCIIByteArray:
if (_inIndex >= _inSize)
{
stream.AddToken( -1 );
return;
}
i = (int)(_inBytes[_inIndex++]);
break;
case TokenSource.CharArray:
if (_inIndex >= _inSize)
{
stream.AddToken( -1 );
return;
}
i = (int)(_inChars[_inIndex++]);
break;
case TokenSource.String:
if (_inIndex >= _inSize)
{
stream.AddToken( -1 );
return;
}
i = (int)(_inString[_inIndex++]);
break;
case TokenSource.NestedStrings:
if (_inNestedSize != 0)
{
if (_inNestedIndex < _inNestedSize)
{
i = _inNestedString[_inNestedIndex++];
break;
}
_inNestedSize = 0;
}
if (_inIndex >= _inSize)
{
stream.AddToken( -1 );
return;
}
i = (int)(_inString[_inIndex++]);
if (i != '{')
break;
for (int istr=0; istr < _searchStrings.Length; istr++)
{
if (0==String.Compare(_searchStrings[istr], 0, _inString, _inIndex-1, _searchStrings[istr].Length, StringComparison.Ordinal))
{
_inNestedString = _replaceStrings[istr];
_inNestedSize = _inNestedString.Length;
_inNestedIndex = 1;
i = _inNestedString[0];
_inIndex += _searchStrings[istr].Length - 1;
break;
}
}
break;
default:
i = _inTokenReader.Read();
if (i == -1)
{
stream.AddToken( -1 );
return;
}
break;
}
if (!inLiteral)
{
switch (i)
{
// skip whitespace
case intSpace:
case intTab:
case intCR:
goto BEGINNING;
// count linefeeds
case intLF:
LineNo++;
goto BEGINNING;
case intOpenBracket:
_inProcessingTag++;
stream.AddToken( bra );
continue;
case intCloseBracket:
_inProcessingTag--;
stream.AddToken( ket );
if (endAfterKet)
return;
continue;
case intEquals:
stream.AddToken( equals );
continue;
case intSlash:
if (_inProcessingTag != 0)
{
stream.AddToken( slash );
continue;
}
break;
case intQuest:
if (_inProcessingTag != 0)
{
stream.AddToken( quest );
continue;
}
break;
case intBang:
if (_inProcessingTag != 0)
{
stream.AddToken( bang );
continue;
}
break;
case intDash:
if (_inProcessingTag != 0)
{
stream.AddToken( dash );
continue;
}
break;
case intQuote:
inLiteral = true;
inQuotedString = true;
goto BEGINNING;
}
}
else
{
switch (i)
{
case intOpenBracket:
if (!inQuotedString)
{
_inSavedCharacter = i;
stream.AddToken( cstr );
stream.AddString( this.GetStringToken() );
continue;
}
break;
case intCloseBracket:
case intEquals:
case intSlash:
if (!inQuotedString && _inProcessingTag != 0)
{
_inSavedCharacter = i;
stream.AddToken( cstr );
stream.AddString( this.GetStringToken() );
continue;
}
break;
case intQuote:
if (inQuotedString)
{
stream.AddToken( cstr );
stream.AddString( this.GetStringToken() );
continue;
}
break;
case intTab:
case intCR:
case intSpace:
if (!inQuotedString)
{
stream.AddToken( cstr );
stream.AddString( this.GetStringToken() );
continue;
}
break;
// count linefeeds
case intLF:
LineNo++;
if (!inQuotedString)
{
stream.AddToken( cstr );
stream.AddString( this.GetStringToken() );
continue;
}
break;
}
}
inLiteral = true;
// add character to the string
if (m._outIndex < StringMaker.outMaxSize)
{
// easy case
m._outChars[m._outIndex++] = (char)i;
}
else
{
if (m._outStringBuilder == null)
{
// OK, first check if we have to init the StringBuilder
m._outStringBuilder = new StringBuilder();
}
// OK, copy from _outChars to _outStringBuilder
m._outStringBuilder.Append(m._outChars, 0, StringMaker.outMaxSize);
// reset _outChars pointer
m._outChars[0] = (char)i;
m._outIndex = 1;
}
goto BEGINNING;
}
}
private int DetermineFormat(TokenizerStream stream)
{
if (stream.GetNextToken() == Tokenizer.bra)
{
if (stream.GetNextToken() == Tokenizer.quest)
{
_t.GetTokens(stream, -1, true);
stream.GoToPosition(2);
bool sawEquals = false;
bool sawEncoding = false;
short i;
for (i = stream.GetNextToken(); i != -1 && i != Tokenizer.ket; i = stream.GetNextToken())
{
switch (i)
{
case Tokenizer.cstr:
if (sawEquals && sawEncoding)
{
_t.ChangeFormat(System.Text.Encoding.GetEncoding(stream.GetNextString()));
return(0);
}
else if (!sawEquals)
{
if (String.Compare(stream.GetNextString(), "encoding", StringComparison.Ordinal) == 0)
{
sawEncoding = true;
}
}
else
{
sawEquals = false;
sawEncoding = false;
stream.ThrowAwayNextString();
}
break;
case Tokenizer.equals:
sawEquals = true;
break;
default:
throw new XmlSyntaxException(_t.LineNo, Environment.GetResourceString("XMLSyntax_UnexpectedEndOfFile"));
}
}
return(0);
}
}
return(2);
}
