internal virtual void AssignLexerTokenTypes(Grammar g, IList<GrammarAST> tokensDefs)
{
Grammar G = g.GetOutermostGrammar(); // put in root, even if imported
foreach (GrammarAST def in tokensDefs)
{
// tokens { id (',' id)* } so must check IDs not TOKEN_REF
if (Grammar.IsTokenName(def.Text))
{
G.DefineTokenName(def.Text);
}
}
/* Define token types for nonfragment rules which do not include a 'type(...)'
* or 'more' lexer command.
*/
foreach (Rule r in g.rules.Values)
{
if (!r.IsFragment() && !HasTypeOrMoreCommand(r))
{
G.DefineTokenName(r.name);
}
}
// FOR ALL X : 'xxx'; RULES, DEFINE 'xxx' AS TYPE X
IList<System.Tuple<GrammarAST, GrammarAST>> litAliases = Grammar.GetStringLiteralAliasesFromLexerRules(g.ast);
ISet<string> conflictingLiterals = new HashSet<string>();
if (litAliases != null)
{
foreach (System.Tuple<GrammarAST, GrammarAST> pair in litAliases)
{
GrammarAST nameAST = pair.Item1;
GrammarAST litAST = pair.Item2;
if (!G.stringLiteralToTypeMap.ContainsKey(litAST.Text))
{
G.DefineTokenAlias(nameAST.Text, litAST.Text);
}
else
{
// oops two literal defs in two rules (within or across modes).
conflictingLiterals.Add(litAST.Text);
}
}
foreach (string lit in conflictingLiterals)
{
// Remove literal if repeated across rules so it's not
// found by parser grammar.
int value;
if (G.stringLiteralToTypeMap.TryGetValue(lit, out value))
{
G.stringLiteralToTypeMap.Remove(lit);
if (value > 0 && value < G.typeToStringLiteralList.Count && lit.Equals(G.typeToStringLiteralList[value]))
{
G.typeToStringLiteralList[value] = null;
}
}
}
}
}
