private void RunTestScenario(string folderWithPositives, string folderWithNegatives, IStride queryStride, int seconds, List<int> startAts)
{
int iterations = startAts.Count;
var positives = AllFiles(folderWithPositives);
var negatives = AllFiles(folderWithNegatives);
for (int iteration = 0; iteration < iterations; ++iteration)
{
OnTestRunnerEvent(
OngoingActionEvent,
new TestRunnerOngoingEventArgs
{
Message =
string.Format(
"Iteration {0} out of {1} with {2}, query seconds {3}",
iteration + 1,
iterations,
queryStride,
seconds)
});
var stopwatch = new Stopwatch();
stopwatch.Start();
int trueNegatives = 0, truePositives = 0, falseNegatives = 0, falsePositives = 0, verified = 0;
var truePositiveHammingDistance = new ConcurrentBag<int>();
var falseNegativesHammingDistance = new ConcurrentBag<int>();
var falsePositivesHammingDistance = new ConcurrentBag<int>();
var sb = TestRunnerWriter.StartTestIteration();
int currentIteration = iteration;
int startAt = startAts[currentIteration];
Parallel.ForEach(
positives,
positive =>
{
Interlocked.Increment(ref verified);
var tags = GetTagsFromFile(positive);
var actualTrack = GetActualTrack(tags);
var queryResult = BuildQuery(queryStride, seconds, positive, startAt).Result;
if (!queryResult.ContainsMatches)
{
Interlocked.Increment(ref falseNegatives);
var notFoundLine = GetNotFoundLine(tags);
AppendLine(sb, notFoundLine);
OnTestRunnerEvent(
PositiveNotFoundEvent,
GetTestRunnerEventArgs(
truePositives,
trueNegatives,
falsePositives,
falseNegatives,
notFoundLine,
verified));
return;
}
var recognizedTrack = queryResult.BestMatch.Track;
bool isSuccessful = recognizedTrack.TrackReference.Equals(actualTrack.TrackReference);
if (isSuccessful)
{
Interlocked.Increment(ref truePositives);
truePositiveHammingDistance.Add(queryResult.BestMatch.HammingSimilaritySum);
}
else
{
Interlocked.Increment(ref falsePositives);
falseNegativesHammingDistance.Add(queryResult.BestMatch.HammingSimilaritySum);
}
var foundLine = GetFoundLine(ToTrackString(actualTrack), recognizedTrack, isSuccessful, queryResult);
AppendLine(sb, foundLine);
OnTestRunnerEvent(PositiveFoundEvent, GetTestRunnerEventArgs(truePositives, trueNegatives, falsePositives, falseNegatives, foundLine, verified));
});
Parallel.ForEach(
negatives,
negative =>
{
Interlocked.Increment(ref verified);
var tags = GetTagsFromFile(negative);
var queryResult = BuildQuery(queryStride, seconds, negative, startAt).Result;
if (!queryResult.ContainsMatches)
{
Interlocked.Increment(ref trueNegatives);
var notFoundLine = GetNotFoundLine(tags);
AppendLine(sb, notFoundLine);
OnTestRunnerEvent(
NegativeNotFoundEvent,
GetTestRunnerEventArgs(
truePositives,
trueNegatives,
falsePositives,
falseNegatives,
notFoundLine,
verified));
return;
}
var recognizedTrack = queryResult.BestMatch.Track;
falsePositivesHammingDistance.Add(queryResult.BestMatch.HammingSimilaritySum);
Interlocked.Increment(ref falsePositives);
var foundLine = GetFoundLine(ToTrackString(tags), recognizedTrack, false, queryResult);
AppendLine(sb, foundLine);
OnTestRunnerEvent(
NegativeFoundEvent,
GetTestRunnerEventArgs(truePositives, trueNegatives, falsePositives, falseNegatives, foundLine, verified));
});
stopwatch.Stop();
var fscore = new FScore(truePositives, trueNegatives, falsePositives, falseNegatives);
var stats = HammingDistanceResultStatistics.From(
truePositiveHammingDistance,
falseNegativesHammingDistance,
falsePositivesHammingDistance,
testRunnerConfig.Percentiles);
TestRunnerWriter.FinishTestIteration(sb, fscore, stats, stopwatch.ElapsedMilliseconds);
TestRunnerWriter.SaveTestIterationToFolder(sb, pathToResultsFolder, queryStride, GetInsertMetadata(), seconds, startAt);
var finishedTestIteration = GetTestRunnerEventArgsForFinishedTestIteration(queryStride, seconds, startAts, fscore, stats, iteration, stopwatch, verified);
OnTestRunnerEvent(TestIterationFinishedEvent, finishedTestIteration);
TestRunnerWriter.AppendLine(suite, finishedTestIteration.RowWithDetails);
}
}
