private bool ConsiderIncreasingBatchSize(int amountOfItemsToIndex, long size, TimeSpan indexingDuration)
{
if (amountOfItemsToIndex < NumberOfItemsToIndexInSingleBatch)
{
return false;
}
if (GetLastAmountOfItems().Any(x => x < NumberOfItemsToIndexInSingleBatch))
{
// this is the first time we hit the limit, we will give another go before we increase
// the batch size
return false;
}
// in the previous run, we also hit the current limit, we need to check if we can increase the max batch size
// here we make the assumptions that the average size of documents are the same. We check if we doubled the amount of memory
// that we used for the last batch (note that this is only an estimate number, but should be close enough), would we still be
// within the limits that governs us
var sizeInMegabytes = size / 1024 / 1024;
// we don't actually *know* what the actual cost of indexing, because that depends on many factors (how the index
// is structured, is it analyzed/default/not analyzed, etc). We just assume for now that it takes 25% of the actual
// on disk structure per each active index. That should give us a good guesstimate about the value.
// Because of the way we are executing indexes, only N are running at once, where N is the parallel level, so we take
// that into account, you may have 10 indexes but only 2 CPUs, so we only consider the cost of executing 2 indexes,
// not all 10
var sizedPlusIndexingCost = sizeInMegabytes * (1 + (0.25 * Math.Min(context.IndexDefinitionStorage.IndexesCount, context.Configuration.MaxNumberOfParallelIndexTasks)));
var remainingMemoryAfterBatchSizeIncrease = MemoryStatistics.AvailableMemory - sizedPlusIndexingCost;
if (remainingMemoryAfterBatchSizeIncrease < context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit)
return false;
// here we assume that the next batch would be 175% as long as the current one
// and there is no point in trying if we are just going to blow out past our max latency
var timeSpan = indexingDuration.Add(TimeSpan.FromMilliseconds(indexingDuration.TotalMilliseconds * 0.75));
if (timeSpan > context.Configuration.MaxIndexingRunLatency)
return false;
NumberOfItemsToIndexInSingleBatch = Math.Min(MaxNumberOfItems,
NumberOfItemsToIndexInSingleBatch*2);
return true;
}