protected override async Task<bool> Fetch(CollectorHttpClient client, ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken)
{
JObject root = await client.GetJObjectAsync(Index, cancellationToken);
IEnumerable<CatalogItem> rootItems = root["items"]
.Select(item => new CatalogItem(item))
.Where(item => item.CommitTimeStamp > front.Value)
.OrderBy(item => item.CommitTimeStamp);
bool acceptNextBatch = false;
foreach (CatalogItem rootItem in rootItems)
{
JObject page = await client.GetJObjectAsync(rootItem.Uri, cancellationToken);
JToken context = null;
page.TryGetValue("@context", out context);
var batches = await CreateBatches(page["items"]
.Select(item => new CatalogItem(item))
.Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value));
var orderedBatches = batches
.OrderBy(batch => batch.CommitTimeStamp)
.ToList();
var lastBatch = orderedBatches.LastOrDefault();
foreach (var batch in orderedBatches)
{
acceptNextBatch = await OnProcessBatch(
client,
batch.Items.Select(item => item.Value),
context,
batch.CommitTimeStamp,
batch.CommitTimeStamp == lastBatch.CommitTimeStamp,
cancellationToken);
front.Value = batch.CommitTimeStamp;
await front.Save(cancellationToken);
Trace.TraceInformation("CommitCatalog.Fetch front.Save has value: {0}", front);
if (!acceptNextBatch)
{
break;
}
}
if (!acceptNextBatch)
{
break;
}
}
return acceptNextBatch;
}