public void ClusterSharding_with_flaky_journal_should_recover_after_journal_failure()
{
ClusterSharding_with_flaky_journal_should_join_cluster();
Within(TimeSpan.FromSeconds(20), () =>
{
RunOn(() =>
{
TestConductor.Blackhole(_controller, _first, ThrottleTransportAdapter.Direction.Both).Wait();
TestConductor.Blackhole(_controller, _second, ThrottleTransportAdapter.Direction.Both).Wait();
}, _controller);
EnterBarrier("journal-backholded");
RunOn(() =>
{
// try with a new shard, will not reply until journal is available again
var region = _region.Value;
region.Tell(new Add("40", 4));
var probe = CreateTestProbe();
region.Tell(new Get("40"), probe.Ref);
probe.ExpectNoMsg(TimeSpan.FromSeconds(1));
}, _first);
EnterBarrier("first-delayed");
RunOn(() =>
{
TestConductor.PassThrough(_controller, _first, ThrottleTransportAdapter.Direction.Both).Wait();
TestConductor.PassThrough(_controller, _second, ThrottleTransportAdapter.Direction.Both).Wait();
}, _controller);
EnterBarrier("journal-ok");
RunOn(() =>
{
var region = _region.Value;
region.Tell(new Get("21"));
ExpectMsg<Value>(v => v.Id == "21" && v.N == 3);
var entity21 = LastSender;
var shard2 = Sys.ActorSelection(entity21.Path.Parent);
//Test the PersistentShardCoordinator allocating shards during a journal failure
region.Tell(new Add("30", 3));
//Test the Shard starting entities and persisting during a journal failure
region.Tell(new Add("11", 1));
//Test the Shard passivate works during a journal failure
shard2.Tell(new Passivate(PoisonPill.Instance), entity21);
region.Tell(new Add("21", 1));
region.Tell(new Get("21"));
ExpectMsg<Value>(v => v.Id == "21" && v.N == 1);
region.Tell(new Get("30"));
ExpectMsg<Value>(v => v.Id == "30" && v.N == 3);
region.Tell(new Get("11"));
ExpectMsg<Value>(v => v.Id == "11" && v.N == 1);
region.Tell(new Get("40"));
ExpectMsg<Value>(v => v.Id == "40" && v.N == 4);
}, _first);
EnterBarrier("verified-first");
RunOn(() =>
{
var region = _region.Value;
region.Tell(new Add("10", 1));
region.Tell(new Add("20", 2));
region.Tell(new Add("30", 3));
region.Tell(new Add("11", 4));
region.Tell(new Get("10"));
ExpectMsg<Value>(v => v.Id == "10" && v.N == 2);
region.Tell(new Get("11"));
ExpectMsg<Value>(v => v.Id == "11" && v.N == 5);
region.Tell(new Get("20"));
ExpectMsg<Value>(v => v.Id == "20" && v.N == 4);
region.Tell(new Get("30"));
ExpectMsg<Value>(v => v.Id == "30" && v.N == 6);
}, _second);
EnterBarrier("after-3");
});
}
}