protected override async Task RunAsync(CancellationToken runAsyncCancellationToken)
{
// This is to keep track of exceptions in the validation step at the end of
// each iteration of the ChaosTestScenario that is being used under the cover
//
bool validationExceptionCaught = false;
IReliableDictionary<string, CurrentState> chaosServiceState =
await this.StateManager.GetOrAddAsync<IReliableDictionary<string, CurrentState>>(StringResource.ChaosServiceStateKey);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
if (!await chaosServiceState.ContainsKeyAsync(tx, StringResource.ChaosServiceStateKey, LockMode.Update))
{
await chaosServiceState.AddAsync(tx, StringResource.ChaosServiceStateKey, CurrentState.Stopped);
}
await tx.CommitAsync();
}
while (!runAsyncCancellationToken.IsCancellationRequested)
{
try
{
// check to see if we're in a "stop" or "start" state.
// this continues to poll until we're in a "start" state.
// a ReliableDictionary is used to store this information so that if the service
// fails over to another node, the state is preserved and the chaos test will continue to execute.
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue<CurrentState> currentStateResult =
await chaosServiceState.TryGetValueAsync(tx, StringResource.ChaosServiceStateKey);
if (currentStateResult.HasValue &&
(currentStateResult.Value == CurrentState.Stopped ||
currentStateResult.Value == CurrentState.None))
{
await Task.Delay(Constants.IntervalBetweenLoopIteration, runAsyncCancellationToken);
continue;
}
}
// this section runs the actual chaos test.
// the cancellation token source is linked to the token provided to RunAsync so that we
// can stop the test if the service needs to shut down.
using (FabricClient fabricClient = new FabricClient())
{
using (this.stopEventTokenSource = CancellationTokenSource.CreateLinkedTokenSource(runAsyncCancellationToken))
{
// when a validation exception is caught, this waits for a while to let the cluster stabilize before continuing.
if (validationExceptionCaught)
{
await Task.Delay(ChaosTestConfigSettings.MaxClusterStabilizationTimeout, this.stopEventTokenSource.Token);
validationExceptionCaught = false;
}
ChaosTestScenarioParameters chaosScenarioParameters =
new ChaosTestScenarioParameters(
ChaosTestConfigSettings.MaxClusterStabilizationTimeout,
ChaosTestConfigSettings.MaxConcurrentFaults,
ChaosTestConfigSettings.EnableMoveReplicaFaults,
TimeSpan.MaxValue)
{
WaitTimeBetweenFaults =
ChaosTestConfigSettings.WaitTimeBetweenFaults,
WaitTimeBetweenIterations =
ChaosTestConfigSettings.WaitTimeBetweenIterations
};
ChaosTestScenario chaosTestScenario = new ChaosTestScenario(fabricClient, chaosScenarioParameters);
// capture progress events so we can report them back
chaosTestScenario.ProgressChanged += this.TestScenarioProgressChanged;
// this continuously runs the chaos test until the CancellationToken is signaled.
await chaosTestScenario.ExecuteAsync(this.stopEventTokenSource.Token);
}
}
}
catch (TimeoutException e)
{
string message = $"Caught TimeoutException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
ServiceEventSource.Current.ServiceMessage(this, message);
validationExceptionCaught = true;
await this.StoreEventAsync(message);
}
catch (FabricValidationException e)
{
string message = $"Caught FabricValidationException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
ServiceEventSource.Current.ServiceMessage(this, message);
validationExceptionCaught = true;
await this.StoreEventAsync(message);
}
catch (OperationCanceledException)
{
if (runAsyncCancellationToken.IsCancellationRequested)
{
// if RunAsync is canceled then we need to quit.
throw;
}
ServiceEventSource.Current.ServiceMessage(
this,
"Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
}
catch (AggregateException e)
{
if (e.InnerException is OperationCanceledException)
{
if (runAsyncCancellationToken.IsCancellationRequested)
{
// if RunAsync is canceled then we need to quit.
throw;
}
ServiceEventSource.Current.ServiceMessage(
this,
"Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
}
else
{
string message = $"Caught unexpected Exception during test excecution {e.InnerException}";
ServiceEventSource.Current.ServiceMessage(this, message);
await this.StoreEventAsync(message);
}
}
catch (Exception e)
{
string message = $"Caught unexpected Exception during test excecution {e}";
ServiceEventSource.Current.ServiceMessage(this, message);
await this.StoreEventAsync(message);
}
}
}
