public IList<Token> ReadTokenList(System.IO.Stream stream)
{
byte[] bs = new byte[0xffff];
stream.Read(bs, 0, 2);
UInt16 len = BitConverter.ToUInt16(bs, 0);
stream.Read(bs, 0, 2);
UInt16 lenstr = BitConverter.ToUInt16(bs, 0);
List<string> strstore = new List<string>();
List<Token> tokens = new List<Token>();
for (int i = 0; i < lenstr; i++)
{
stream.Read(bs, 0, 2);
UInt16 slen = BitConverter.ToUInt16(bs, 0);
stream.Read(bs, 0, slen);
strstore.Add(System.Text.Encoding.UTF8.GetString(bs, 0, slen));
}
for (int i = 0; i < len; i++)
{
Token t = new Token();
stream.Read(bs, 0, 4);
UInt32 type = BitConverter.ToUInt32(bs, 0);
t.type = (TokenType)(type % 0x0100);
t.text = strstore[(int)(type / 0x100)];
tokens.Add(t);
}
if (stream.Position < stream.Length)
{
UInt16 linecount = 0;
byte[] bufu = new byte[2];
stream.Read(bufu, 0, 2);
linecount = BitConverter.ToUInt16(bufu, 0);
UInt16[] linetoken = new UInt16[linecount];
for (int i = 0; i < linecount; i++)
{
stream.Read(bufu, 0, 2);
linetoken[i] = BitConverter.ToUInt16(bufu, 0);
}
//int token = 0;
for (int i = 0; i < linecount; i++)
{
if ((i + 1) < linecount && linetoken[i + 1] == linetoken[i]) continue;
if ((i + 1) < linecount)
{
for (int j = linetoken[i]; j < linetoken[i + 1]; j++)
{
var t = tokens[j];
t.line = i + 1;
tokens[j] = t;
}
}
else
{
for (int j = linetoken[i]; j < tokens.Count; j++)
{
var t = tokens[j];
t.line = i + 1;
tokens[j] = t;
}
}
//token = linetoken[i];
}
}
return tokens;
}
}