internal virtual Template GetTokenVocabOutput()
{
Template vocabFileST = new Template(vocabFilePattern);
IDictionary<string, int> tokens = new LinkedHashMap<string, int>();
// make constants for the token names
foreach (string t in g.tokenNameToTypeMap.Keys)
{
int tokenType;
if (g.tokenNameToTypeMap.TryGetValue(t, out tokenType) && tokenType >= TokenConstants.MinUserTokenType)
{
tokens[t] = tokenType;
}
}
vocabFileST.Add("tokens", tokens);
// now dump the strings
IDictionary<string, int> literals = new LinkedHashMap<string, int>();
foreach (string literal in g.stringLiteralToTypeMap.Keys)
{
int tokenType;
if (g.stringLiteralToTypeMap.TryGetValue(literal, out tokenType) && tokenType >= TokenConstants.MinUserTokenType)
{
literals[literal] = tokenType;
}
}
vocabFileST.Add("literals", literals);
return vocabFileST;
}