public void SaveTokenList(IList<Token> tokens, System.IO.Stream stream)
{
Dictionary<string, UInt32> strs = new Dictionary<string, UInt32>();
List<UInt16> lines = new List<ushort>();
List<string> strstore = new List<string>();
List<UInt32> strindex = new List<UInt32>();
if (tokens.Count > 0xffff)
{
throw new Exception("不支持这么复杂的token保存");
}
byte[] bs = BitConverter.GetBytes((UInt16)tokens.Count);
stream.Write(bs, 0, bs.Length);
int index = 0;
foreach (var t in tokens)
{
UInt32 type = (UInt32)t.type;
if (strs.ContainsKey(t.text) == false)
{
strstore.Add(t.text);
strs[t.text] = (uint)(strs.Count * 0x0100);
}
type += strs[t.text];
strindex.Add(type);
int line = t.line - 1;
while (line >= lines.Count)
lines.Add((UInt16)index);
index++;
}
byte[] bsstr = BitConverter.GetBytes((UInt16)strstore.Count);
stream.Write(bsstr, 0, bsstr.Length);
foreach (var s in strstore)
{
byte[] sbs = System.Text.Encoding.UTF8.GetBytes(s);
byte[] sbslen = BitConverter.GetBytes((UInt16)sbs.Length);
stream.Write(sbslen, 0, sbslen.Length);
stream.Write(sbs, 0, sbs.Length);
}
foreach (var i in strindex)
{
byte[] nbs = BitConverter.GetBytes((UInt32)i);
stream.Write(nbs, 0, nbs.Length);
}
UInt16 linecount = (UInt16)lines.Count;
stream.Write(BitConverter.GetBytes(linecount), 0, 2);
foreach (UInt16 lstarttoken in lines)
{
byte[] nbs = BitConverter.GetBytes(lstarttoken);
stream.Write(nbs, 0, 2);
}
}
public IList<Token> ReadTokenList(System.IO.Stream stream)
