Raven.Database.Prefetching.PrefetchingBehavior.FilterDocuments C# (CSharp) Method

		public bool FilterDocuments(JsonDocument document)
		{
			HashSet<Etag> etags;
			return (documentsToRemove.TryGetValue(document.Key, out etags) && etags.Contains(document.Etag)) == false;
		}

        private JsonDocumentsToReplicate GetJsonDocuments(SourceReplicationInformationWithBatchInformation destinationsReplicationInformationForSource, ReplicationStrategy destination, PrefetchingBehavior prefetchingBehavior, ReplicationStatisticsRecorder.ReplicationStatisticsRecorderScope scope)
        {
            var timeout = TimeSpan.FromSeconds(docDb.Configuration.Replication.FetchingFromDiskTimeoutInSeconds);
            var duration = Stopwatch.StartNew();
            var result = new JsonDocumentsToReplicate
            {
                LastEtag = Etag.Empty,
            };
            try
            {
                var destinationId = destinationsReplicationInformationForSource.ServerInstanceId.ToString();
                var maxNumberOfItemsToReceiveInSingleBatch = destinationsReplicationInformationForSource.MaxNumberOfItemsToReceiveInSingleBatch;

                docDb.TransactionalStorage.Batch(actions =>
                {
                    var lastEtag = destinationsReplicationInformationForSource.LastDocumentEtag;

                    int docsSinceLastReplEtag = 0;
                    List<JsonDocument> docsToReplicate;
                    List<JsonDocument> filteredDocsToReplicate;
                    result.LastEtag = lastEtag;

                    while (true)
                    {
                        docDb.WorkContext.CancellationToken.ThrowIfCancellationRequested();

                        docsToReplicate = GetDocsToReplicate(actions, prefetchingBehavior, result, maxNumberOfItemsToReceiveInSingleBatch);

                        filteredDocsToReplicate =
                            docsToReplicate
                                .Where(document =>
                                {
                                    var info = docDb.Documents.GetRecentTouchesFor(document.Key);
                                    if (info != null)
                                    {
                                        if (info.TouchedEtag.CompareTo(result.LastEtag) > 0)
                                        {
                                            log.Debug(
                                                "Will not replicate document '{0}' to '{1}' because the updates after etag {2} are related document touches",
                                                document.Key, destinationId, info.TouchedEtag);
                                            return false;
                                        }
                                    }

                                    string reason;
                                    return destination.FilterDocuments(destinationId, document.Key, document.Metadata, out reason) &&
                                           prefetchingBehavior.FilterDocuments(document);
                                })
                                .ToList();

                        docsSinceLastReplEtag += docsToReplicate.Count;
                        result.CountOfFilteredDocumentsWhichAreSystemDocuments +=
                            docsToReplicate.Count(doc => destination.IsSystemDocumentId(doc.Key));
                        result.CountOfFilteredDocumentsWhichOriginFromDestination +=
                            docsToReplicate.Count(doc => destination.OriginsFromDestination(destinationId, doc.Metadata));

                        if (docsToReplicate.Count > 0)
                        {
                            var lastDoc = docsToReplicate.Last();
                            Debug.Assert(lastDoc.Etag != null);
                            result.LastEtag = lastDoc.Etag;
                            if (lastDoc.LastModified.HasValue)
                                result.LastLastModified = lastDoc.LastModified.Value;
                        }

                        if (docsToReplicate.Count == 0 || filteredDocsToReplicate.Count != 0)
                        {
                            break;
                        }

                        log.Debug("All the docs were filtered, trying another batch from etag [>{0}]", result.LastEtag);

                        if (duration.Elapsed > timeout)
                            break;
                    }

                    log.Debug(() =>
                    {
                        if (docsSinceLastReplEtag == 0)
                            return string.Format("No documents to replicate to {0} - last replicated etag: {1}", destination,
                                lastEtag);

                        if (docsSinceLastReplEtag == filteredDocsToReplicate.Count)
                            return string.Format("Replicating {0} docs [>{1}] to {2}.",
                                docsSinceLastReplEtag,
                                lastEtag,
                                destination);

                        var diff = docsToReplicate.Except(filteredDocsToReplicate).Select(x => x.Key);
                        return string.Format("Replicating {1} docs (out of {0}) [>{4}] to {2}. [Not replicated: {3}]",
                            docsSinceLastReplEtag,
                            filteredDocsToReplicate.Count,
                            destination,
                            string.Join(", ", diff),
                            lastEtag);
                    });

                    scope.Record(new RavenJObject
                    {
                        {"StartEtag", lastEtag.ToString()},
                        {"EndEtag", result.LastEtag.ToString()},
                        {"Count", docsSinceLastReplEtag},
                        {"FilteredCount", filteredDocsToReplicate.Count}
                    });

                    result.LoadedDocs = filteredDocsToReplicate;
                    docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeMetric(destination).Mark(docsSinceLastReplEtag);
                    docDb.WorkContext.MetricsCounters.GetReplicationBatchSizeHistogram(destination).Update(docsSinceLastReplEtag);

                    result.Documents = new RavenJArray(filteredDocsToReplicate
                        .Select(x =>
                        {
                            JsonDocument.EnsureIdInMetadata(x);
                            EnsureReplicationInformationInMetadata(x.Metadata, docDb);
                            return x;
                        })
                        .Select(x => x.ToJson()));
                });
            }
            catch (Exception e)
            {
                scope.RecordError(e);
                log.WarnException(
                    "Could not get documents to replicate after: " +
                    destinationsReplicationInformationForSource.LastDocumentEtag, e);
            }
            return result;
        }