[System.Security.SecurityCritical] // auto-generated
internal bool Tokenize(TokenType TokenMask, out TokenType tokenType, out int tokenValue, ref __DTString str) {
tokenType = TokenType.UnknownToken;
tokenValue = 0;
TokenHashValue value;
Contract.Assert(str.Index < str.Value.Length, "DateTimeFormatInfo.Tokenize(): start < value.Length");
char ch = str.m_current;
bool isLetter = Char.IsLetter(ch);
if (isLetter) {
ch = Char.ToLower(ch, this.Culture);
if (IsHebrewChar(ch) && TokenMask == TokenType.RegularTokenMask) {
bool badFormat;
if (TryParseHebrewNumber(ref str, out badFormat, out tokenValue)) {
if (badFormat) {
tokenType = TokenType.UnknownToken;
return (false);
}
// This is a Hebrew number.
// Do nothing here. TryParseHebrewNumber() will update token accordingly.
tokenType = TokenType.HebrewNumber;
return (true);
}
}
}
int hashcode = ch % TOKEN_HASH_SIZE;
int hashProbe = 1 + ch % SECOND_PRIME;
int remaining = str.len - str.Index;
int i = 0;
TokenHashValue[] hashTable = m_dtfiTokenHash;
if (hashTable == null) {
hashTable = CreateTokenHashTable();
}
do {
value = hashTable[hashcode];
if (value == null) {
// Not found.
break;
}
// Check this value has the right category (regular token or separator token) that we are looking for.
if (((int)value.tokenType & (int)TokenMask) > 0 && value.tokenString.Length <= remaining) {
if (String.Compare(str.Value, str.Index, value.tokenString, 0, value.tokenString.Length, this.Culture, CompareOptions.IgnoreCase)==0) {
if (isLetter) {
// If this token starts with a letter, make sure that we won't allow partial match. So you can't tokenize "MarchWed" separately.
int nextCharIndex;
if ((nextCharIndex = str.Index + value.tokenString.Length) < str.len) {
// Check word boundary. The next character should NOT be a letter.
char nextCh = str.Value[nextCharIndex];
if (Char.IsLetter(nextCh)) {
return (false);
}
}
}
tokenType = value.tokenType & TokenMask;
tokenValue = value.tokenValue;
str.Advance(value.tokenString.Length);
return (true);
} else if (value.tokenType == TokenType.MonthToken && HasSpacesInMonthNames) {
// For month token, we will match the month names which have spaces.
int matchStrLen = 0;
if (str.MatchSpecifiedWords(value.tokenString, true, ref matchStrLen)) {
tokenType = value.tokenType & TokenMask;
tokenValue = value.tokenValue;
str.Advance(matchStrLen);
return (true);
}
} else if (value.tokenType == TokenType.DayOfWeekToken && HasSpacesInDayNames) {
// For month token, we will match the month names which have spaces.
int matchStrLen = 0;
if (str.MatchSpecifiedWords(value.tokenString, true, ref matchStrLen)) {
tokenType = value.tokenType & TokenMask;
tokenValue = value.tokenValue;
str.Advance(matchStrLen);
return (true);
}
}
}
i++;
hashcode += hashProbe;
if (hashcode >= TOKEN_HASH_SIZE) hashcode -= TOKEN_HASH_SIZE;
}while (i < TOKEN_HASH_SIZE);
return (false);
}