public int GetNextToken()
{
int zzInput;
int zzAction;
// cached fields:
int zzCurrentPosL;
int zzMarkedPosL;
int zzEndReadL = zzEndRead;
char[] zzBufferL = zzBuffer;
char[] zzCMapL = ZZ_CMAP;
int[] zzTransL = ZZ_TRANS;
int[] zzRowMapL = ZZ_ROWMAP;
int[] zzAttrL = ZZ_ATTRIBUTE;
while (true)
{
zzMarkedPosL = zzMarkedPos;
yyChar += zzMarkedPosL - zzStartRead;
zzAction = -1;
zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
zzState = ZZ_LEXSTATE[zzLexicalState];
// set up zzAction for empty match case:
int zzAttributes = zzAttrL[zzState];
if ((zzAttributes & 1) == 1)
{
zzAction = zzState;
}
{
while (true)
{
if (zzCurrentPosL < zzEndReadL)
{
zzInput = zzBufferL[zzCurrentPosL++];
}
else if (zzAtEOF)
{
zzInput = StandardTokenizerInterface_Fields.YYEOF;
goto zzForActionBreak;
}
else
{
// store back cached positions
zzCurrentPos = zzCurrentPosL;
zzMarkedPos = zzMarkedPosL;
bool eof = ZzRefill();
// get translated positions and possibly new buffer
zzCurrentPosL = zzCurrentPos;
zzMarkedPosL = zzMarkedPos;
zzBufferL = zzBuffer;
zzEndReadL = zzEndRead;
if (eof)
{
zzInput = StandardTokenizerInterface_Fields.YYEOF;
goto zzForActionBreak;
}
else
{
zzInput = zzBufferL[zzCurrentPosL++];
}
}
int zzNext = zzTransL[zzRowMapL[zzState] + zzCMapL[zzInput]];
if (zzNext == -1)
{
goto zzForActionBreak;
}
zzState = zzNext;
zzAttributes = zzAttrL[zzState];
if ((zzAttributes & 1) == 1)
{
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
if ((zzAttributes & 8) == 8)
{
goto zzForActionBreak;
}
}
}
}
zzForActionBreak:
// store back cached position
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction])
{
case 1:
{ // Break so we don't hit fall-through warning:
break; // Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it.
}
// goto case 9; // unreachable
case 9:
break;
case 2:
{
return WORD_TYPE;
}
case 10:
break;
case 3:
{
return NUMERIC_TYPE;
}
case 11:
break;
case 4:
{
return KATAKANA_TYPE;
}
case 12:
break;
case 5:
{
return SOUTH_EAST_ASIAN_TYPE;
}
case 13:
break;
case 6:
{
return IDEOGRAPHIC_TYPE;
}
case 14:
break;
case 7:
{
return HIRAGANA_TYPE;
}
case 15:
break;
case 8:
{
return HANGUL_TYPE;
}
case 16:
break;
default:
if (zzInput == StandardTokenizerInterface_Fields.YYEOF && zzStartRead == zzCurrentPos)
{
zzAtEOF = true;
{
return StandardTokenizerInterface_Fields.YYEOF;
}
}
else
{
ZzScanError(ZZ_NO_MATCH);
}
break;
}
}
}
} /*
* (non-Javadoc)
*
* @see Lucene.Net.Analysis.TokenStream#next()
*/
public override Token Next(/* in */ Token reusableToken)
{
System.Diagnostics.Debug.Assert(reusableToken != null);
int posIncr = 1;
while (true)
{
int tokenType = scanner.GetNextToken();
if (tokenType == StandardTokenizerImpl.YYEOF)
{
return(null);
}
if (scanner.Yylength() <= maxTokenLength)
{
reusableToken.Clear();
reusableToken.SetPositionIncrement(posIncr);
scanner.GetText(reusableToken);
int start = scanner.Yychar();
reusableToken.SetStartOffset(start);
reusableToken.SetEndOffset(start + reusableToken.TermLength());
// This 'if' should be removed in the next release. For now, it converts
// invalid acronyms to HOST. When removed, only the 'else' part should
// remain.
if (tokenType == StandardTokenizerImpl.ACRONYM_DEP)
{
if (replaceInvalidAcronym)
{
reusableToken.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.HOST]);
reusableToken.SetTermLength(reusableToken.TermLength() - 1); // remove extra '.'
}
else
{
reusableToken.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.ACRONYM]);
}
}
else
{
reusableToken.SetType(StandardTokenizerImpl.TOKEN_TYPES[tokenType]);
}
return(reusableToken);
}
// When we skip a too-long term, we still increment the
// position increment
else
{
posIncr++;
}
}
}