| public QuickScanToken ( bool allowLeadingMultilineTrivia ) : QuickScanResult | ||
| allowLeadingMultilineTrivia | bool | |
| return | QuickScanResult | |
public QuickScanResult QuickScanToken(bool allowLeadingMultilineTrivia)
{
AccumulatorState state = allowLeadingMultilineTrivia ? AccumulatorState.InitialAllowLeadingMultilineTrivia : AccumulatorState.Initial;
var offset = _lineBufferOffset;
var index = _lineBufferOffset;
var qtStart = index;
var limit = index + Math.Min(MAXTOKENSIZE, _bufferLen - offset);
int hashCode = Hash.FnvOffsetBias;
byte terminatorLength = 0;
int unicodeValue = 0;
while (index < limit)
{
var c = buffer[index];
// Get the flags for that character.
unicodeValue = (int)(c);
if (unicodeValue >= CHARPROP_LENGTH)
{
continue;
}
var flags = charProperties[unicodeValue];
// Advance the scanner state.
switch (state)
{
case AccumulatorState.InitialAllowLeadingMultilineTrivia:
if (flags == CharFlags.Letter)
{
state = AccumulatorState.Ident;
}
else if (flags == CharFlags.Punct)
{
state = AccumulatorState.Punctuation;
}
else if (flags == CharFlags.CompoundPunctStart)
{
state = AccumulatorState.CompoundPunctStart;
}
else if ((flags & (CharFlags.White | CharFlags.CR | CharFlags.LF)) != 0)
{
}
else
{
state = AccumulatorState.Bad;
continue;
}
break;
case AccumulatorState.Initial:
if (flags == CharFlags.Letter)
{
state = AccumulatorState.Ident;
}
else if (flags == CharFlags.Punct)
{
state = AccumulatorState.Punctuation;
}
else if (flags == CharFlags.CompoundPunctStart)
{
state = AccumulatorState.CompoundPunctStart;
}
else if (flags == CharFlags.White)
{
}
else
{
state = AccumulatorState.Bad;
continue;
}
break;
case AccumulatorState.Ident:
if ((flags & (CharFlags.Letter | CharFlags.IdentOnly | CharFlags.Digit)) != 0)
{
}
else if (flags == CharFlags.White)
{
state = AccumulatorState.FollowingWhite;
}
else if (flags == CharFlags.CR)
{
state = AccumulatorState.CR;
}
else if (flags == CharFlags.LF)
{
terminatorLength = 1;
state = AccumulatorState.Done;
continue;
}
else if (flags == CharFlags.TypeChar)
{
state = AccumulatorState.TypeChar;
}
else if (flags == CharFlags.Punct)
{
state = AccumulatorState.Done;
continue;
}
else
{
state = AccumulatorState.Bad;
continue;
}
break;
case AccumulatorState.TypeChar:
if (flags == CharFlags.White)
{
state = AccumulatorState.FollowingWhite;
}
else if (flags == CharFlags.CR)
{
state = AccumulatorState.CR;
}
else if (flags == CharFlags.LF)
{
terminatorLength = 1;
state = AccumulatorState.Done;
continue;
}
else if ((flags & (CharFlags.Punct | CharFlags.Digit | CharFlags.TypeChar)) != 0)
{
state = AccumulatorState.Done;
continue;
}
else
{
state = AccumulatorState.Bad;
continue;
}
break;
case AccumulatorState.FollowingWhite:
if (flags == CharFlags.White)
{
}
else if (flags == CharFlags.CR)
{
state = AccumulatorState.CR;
}
else if (flags == CharFlags.LF)
{
terminatorLength = 1;
state = AccumulatorState.Done;
continue;
}
else if ((flags & (CharFlags.Complex | CharFlags.IdentOnly)) != 0)
{
state = AccumulatorState.Bad;
continue;
}
else
{
state = AccumulatorState.Done;
continue;
}
break;
case AccumulatorState.Punctuation:
if (flags == CharFlags.White)
{
state = AccumulatorState.FollowingWhite;
}
else if (flags == CharFlags.CR)
{
state = AccumulatorState.CR;
}
else if (flags == CharFlags.LF)
{
terminatorLength = 1;
state = AccumulatorState.Done;
continue;
}
else if ((flags & (CharFlags.Letter | CharFlags.Punct)) != 0)
{
state = AccumulatorState.Done;
continue;
}
else
{
state = AccumulatorState.Bad;
continue;
}
break;
case AccumulatorState.CompoundPunctStart:
if (flags == CharFlags.White)
{
}
else if ((flags & (CharFlags.Letter | CharFlags.Digit)) != 0)
{
state = AccumulatorState.Done;
continue;
}
else
{
state = AccumulatorState.Bad;
continue;
}
break;
case AccumulatorState.CR:
if (flags == CharFlags.LF)
{
terminatorLength = 2;
state = AccumulatorState.Done;
continue;
}
else
{
state = AccumulatorState.Bad;
}
continue;
default:
Debug.Assert(false, "should not get here");
break;
}
index = 1;
//FNV-like hash should work here
//since these strings are short and mostly ASCII
hashCode = (hashCode ^ unicodeValue) * Hash.FnvPrime;
}
if (state == AccumulatorState.Done && terminatorLength == 0)
{
if (terminatorLength != 0)
{
index = 1;
hashCode = (hashCode ^ unicodeValue) * Hash.FnvPrime;
}
return new QuickScanResult(qtStart, index - qtStart, new char[0], hashCode, terminatorLength);
}
else
{
return default(QuickScanResult);
}
}