private bool ConsiderDecreasingBatchSize(int amountOfItemsToIndex, TimeSpan indexingDuration)
{
if (
// we had as much work to do as we are currently capable of handling,
// we might need to increase, but certainly not decrease the batch size
amountOfItemsToIndex >= NumberOfItemsToIndexInSingleBatch ||
// we haven't gone over the max latency limit, no reason to decrease yet
indexingDuration < context.Configuration.MaxIndexingRunLatency)
{
return false;
}
if ((SystemTime.UtcNow - lastIncrease).TotalMinutes < 3)
return true;
// we didn't have a lot of work to do, so let us see if we can reduce the batch size
// we are at the configured minimum, nothing to do
if (NumberOfItemsToIndexInSingleBatch == InitialNumberOfItems)
return true;
// we were above the max/2 the last few times, we can't reduce the work load now
if (GetLastAmountOfItems().Any(x => x > NumberOfItemsToIndexInSingleBatch/2))
return true;
var old = NumberOfItemsToIndexInSingleBatch;
// we have had a couple of times were we didn't get to the current max, so we can probably
// reduce the max again now, this will reduce the memory consumption eventually, and will cause
// faster indexing times in case we get a big batch again
NumberOfItemsToIndexInSingleBatch = Math.Max(InitialNumberOfItems,
NumberOfItemsToIndexInSingleBatch / 2);
// we just reduced the batch size because we have two concurrent runs where we had
// less to do than the previous runs. That indicate the the busy period is over, maybe we
// run out of data? Or the rate of data entry into the system was just reduce?
// At any rate, there is a strong likelyhood of having a lot of garbage in the system
// let us ask the GC nicely to clean it
// but we only want to do it if the change was significant
if (NumberOfItemsToIndexInSingleBatch - old > 4096)
{
GC.Collect(1, GCCollectionMode.Optimized);
}
return true;
}
