private void MaybeAddFutureBatch(List<JsonDocument> past)
{
if (context.Configuration.DisableDocumentPreFetchingForIndexing || context.RunIndexing == false)
return;
if (context.Configuration.MaxNumberOfParallelIndexTasks == 1)
return;
if (past.Count == 0)
return;
if (futureIndexBatches.Count > 5) // we limit the number of future calls we do
{
int alreadyLoaded = futureIndexBatches.Values.Sum(x =>
{
if (x.Task.IsCompleted)
return x.Task.Result.Count;
return autoTuner.NumberOfItemsToIndexInSingleBatch / 4 * 3;
});
if (alreadyLoaded > autoTuner.NumberOfItemsToIndexInSingleBatch)
return;
}
// ensure we don't do TOO much future caching
if (MemoryStatistics.AvailableMemory <
context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit)
return;
// we loaded the maximum amount, there are probably more items to read now.
Etag highestLoadedEtag = GetHighestEtag(past);
Etag nextEtag = GetNextDocumentEtagFromDisk(highestLoadedEtag);
if (nextEtag == highestLoadedEtag)
return; // there is nothing newer to do
if (futureIndexBatches.ContainsKey(nextEtag)) // already loading this
return;
var futureBatchStat = new FutureBatchStats
{
Timestamp = SystemTime.UtcNow,
};
Stopwatch sp = Stopwatch.StartNew();
context.AddFutureBatch(futureBatchStat);
futureIndexBatches.TryAdd(nextEtag, new FutureIndexBatch
{
StartingEtag = nextEtag,
Age = Interlocked.Increment(ref currentIndexingAge),
Task = Task.Factory.StartNew(() =>
{
List<JsonDocument> jsonDocuments = null;
int localWork = 0;
while (context.RunIndexing)
{
jsonDocuments = GetJsonDocsFromDisk(EtagUtil.Increment(nextEtag, -1), null);
if (jsonDocuments.Count > 0)
break;
futureBatchStat.Retries++;
context.WaitForWork(TimeSpan.FromMinutes(10), ref localWork, "PreFetching");
}
futureBatchStat.Duration = sp.Elapsed;
futureBatchStat.Size = jsonDocuments == null ? 0 : jsonDocuments.Count;
if (jsonDocuments != null)
{
MaybeAddFutureBatch(jsonDocuments);
}
return jsonDocuments;
})
});
}
