public void An_actor_watching_a_remote_actor_in_the_cluster_must_be_able_to_shutdown_system_when_using_remote_deployed_actor_on_node_that_crashed()
{
Within(TimeSpan.FromSeconds(20), () =>
{
// fourth actor system will be shutdown, not part of testConductor any more
// so we can't use barriers to synchronize with it
var firstAddress = GetAddress(_config.First);
RunOn(() =>
{
Sys.ActorOf(Props.Create(() => new EndActor(TestActor, null)), "end");
}, _config.First);
EnterBarrier("end-actor-created");
RunOn(() =>
{
var hello = Sys.ActorOf(BlackHoleActor.Props, "hello");
Assert.IsType<RemoteActorRef>(hello);
hello.Path.Address.ShouldBe(GetAddress(_config.First));
Watch(hello);
EnterBarrier("hello-deployed");
MarkNodeAsUnavailable(GetAddress(_config.First));
AwaitAssert(() => ClusterView.UnreachableMembers.Select(x => x.Address).Contains(GetAddress(_config.First)).ShouldBeTrue());
Cluster.Down(GetAddress(_config.First));
// removed
AwaitAssert(() => Assert.False(ClusterView.UnreachableMembers.Select(x => x.Address).Contains(GetAddress(_config.First))));
AwaitAssert(() => Assert.False(ClusterView.Members.Select(x => x.Address).Contains(GetAddress(_config.First))));
ExpectTerminated(hello);
EnterBarrier("first-unavailable");
var timeout = RemainingOrDefault;
try
{
if (!Sys.WhenTerminated.Wait(timeout)) // TestConductor.Shutdown called by First MUST terminate this actor system
{
Assert.True(false, String.Format("Failed to stop [{0}] within [{1}]", Sys.Name, timeout));
}
}
catch (TimeoutException)
{
Assert.True(false, String.Format("Failed to stop [{0}] within [{1}]", Sys.Name, timeout));
}
// signal to the first node that the fourth node is done
var endSystem = ActorSystem.Create("EndSystem", Sys.Settings.Config);
try
{
var endProbe = CreateTestProbe(endSystem);
var endActor = endSystem.ActorOf(Props.Create(() => new EndActor(endProbe.Ref, firstAddress)),
"end");
endActor.Tell(EndActor.SendEnd.Instance);
endProbe.ExpectMsg<EndActor.EndAck>();
}
finally
{
Shutdown(endSystem, TimeSpan.FromSeconds(10));
}
// no barrier here, because it is not part of TestConductor roles any more
}, _config.Fourth);
RunOn(() =>
{
EnterBarrier("hello-deployed");
EnterBarrier("first-unavailable");
// don't end the test until fourth is done
RunOn(() =>
{
// fourth system will be shutdown, remove to not participate in barriers any more
TestConductor.Shutdown(_config.Fourth).Wait();
ExpectMsg<EndActor.End>();
}, _config.First);
EnterBarrier("after-4");
}, _config.First, _config.Second, _config.Third, _config.Fifth);
});
}