private void ProcessDigit(ITextToken tok, int i)
{
m_puncts.Add(new PunctuationToken(PunctuationTokenType.number, null, false, false));
#if UNUSED
// special case: treat a sequence like
// number/punctuation/number
// as if the punctuation were not there. an example of this would be 1:2
// this allows the : in 1:2 not to be counted as punctuation
if (tokens.Count >= 3)
{
// If the last three tokens are number/select punctuation/number
if (tokens[tokens.Count - 3].TokenType == PunctuationTokenType.number)
{
string separator = tokens[tokens.Count - 2].ToString();
//! make the list of separator characters configurable
if (separator == "," || separator == "." || separator == "-" || separator == ":")
{
tokens.RemoveAt(tokens.Count - 2);
// The offset (-2) stays the same as the line of code above
// since after the previous line is executed some of the tokens shift position.
tokens.RemoveAt(tokens.Count - 2);
}
}
}
#endif
}