private void ProcessPunctuation(ITextToken tok, int i)
{
TextTokenSubstring tts = new TextTokenSubstring(tok, i, 1);
bool isInitial = m_quotationCategorizer.IsInitialPunctuation(tts.Text);
bool isFinal = m_quotationCategorizer.IsFinalPunctuation(tts.Text);
m_puncts.Add(new PunctuationToken(PunctuationTokenType.punctuation, tts, isInitial, isFinal));
// special case: treat a sequence like
// opening quotation punctuation/space/opening quotation punctuation
// as if the space were not there. an example of this would be
// U+201C LEFT DOUBLE QUOTATION MARK
// U+0020 SPACE
// U+2018 LEFT SINGLE QUOTATION MARK
// this allows a quotation mark to be considered word initial even if it is followed by a space
if (m_puncts.Count >= 3)
{
// If the last three tokens are punctuation/whitespace/punctuation
if (m_puncts[m_puncts.Count - 2].TokenType == PunctuationTokenType.whitespace &&
!m_puncts[m_puncts.Count - 2].IsParaBreak &&
m_puncts[m_puncts.Count - 3].TokenType == PunctuationTokenType.punctuation)
{
// And both punctuation have quote directions which point in the same direction,
if (m_puncts[m_puncts.Count - 3].IsInitial && m_puncts[m_puncts.Count - 1].IsInitial ||
m_puncts[m_puncts.Count - 3].IsFinal && m_puncts[m_puncts.Count - 1].IsFinal)
{
// THEN mark the whitespace as a quote separator.
m_puncts[m_puncts.Count - 2].TokenType = PunctuationTokenType.quoteSeparator;
}
}
}
}
