public void ClusterHeartbeatSenderState_must_behave_correctly_for_random_operations()
{
var rnd = ThreadLocalRandom.Current;
var nodes =
Enumerable.Range(1, rnd.Next(10, 200))
.Select(n => new UniqueAddress(new Address("akka.tcp", "sys", "n" + n, 2552), n))
.ToList();
Func<UniqueAddress> rndNode = () => nodes[rnd.Next(0, nodes.Count)];
var selfUniqueAddress = rndNode();
var state = EmptyState(selfUniqueAddress);
const int Add = 0;
const int Remove = 1;
const int Unreachable = 2;
const int HeartbeatRsp = 3;
for (var i = 1; i <= 100000; i++)
{
var operation = rnd.Next(Add, HeartbeatRsp + 1);
var node = rndNode();
try
{
switch (operation)
{
case Add:
if (node != selfUniqueAddress && !state.Ring.NodeRing.Contains(node))
{
var oldUnreachable = state.Unreachable;
state = state.AddMember(node);
//keep unreachable
(oldUnreachable.Except(state.ActiveReceivers)).ShouldBe(ImmutableHashSet.Create<UniqueAddress>());
state.FailureDetector.IsMonitoring(node.Address).ShouldBeFalse();
state.FailureDetector.IsAvailable(node.Address).ShouldBeTrue();
}
break;
case Remove:
if (node != selfUniqueAddress && state.Ring.NodeRing.Contains(node))
{
var oldUnreachable = state.Unreachable;
state = state.RemoveMember(node);
// keep unreachable, unless it was the removed
if(oldUnreachable.Contains(node))
oldUnreachable.Except(state.ActiveReceivers).ShouldBe(ImmutableHashSet.Create(node));
else
(oldUnreachable.Except(state.ActiveReceivers)).ShouldBe(ImmutableHashSet.Create<UniqueAddress>());
state.FailureDetector.IsMonitoring(node.Address).ShouldBeFalse();
state.FailureDetector.IsAvailable(node.Address).ShouldBeTrue();
Assert.False(state.ActiveReceivers.Any(x => x == node));
}
break;
case Unreachable:
if (node != selfUniqueAddress && state.ActiveReceivers.Contains(node))
{
state.FailureDetector.Heartbeat(node.Address); //make sure the FD is created
Fd(state, node).MarkNodeAsUnavailable();
state.FailureDetector.IsMonitoring(node.Address).ShouldBeTrue();
state.FailureDetector.IsAvailable(node.Address).ShouldBeFalse();
}
break;
case HeartbeatRsp:
if (node != selfUniqueAddress && state.Ring.NodeRing.Contains(node))
{
var oldUnreachable = state.Unreachable;
var oldReceivers = state.ActiveReceivers;
var oldRingReceivers = state.Ring.MyReceivers.Value;
state = state.HeartbeatRsp(node);
if(oldUnreachable.Contains(node))
Assert.False(state.Unreachable.Contains(node));
if(oldUnreachable.Contains(node) && !oldRingReceivers.Contains(node))
state.FailureDetector.IsMonitoring(node.Address).ShouldBeFalse();
if(oldRingReceivers.Contains(node))
state.FailureDetector.IsMonitoring(node.Address).ShouldBeTrue();
state.Ring.MyReceivers.Value.ShouldBe(oldRingReceivers);
state.FailureDetector.IsAvailable(node.Address).ShouldBeTrue();
}
break;
}
}
catch (Exception)
{
Debug.WriteLine("Failure context: i = {0}, node = {1}, op={2}, unreachable={3}, ringReceivers={4}, ringNodes={5}", i, node, operation,
string.Join(",",state.Unreachable),
string.Join(",", state.Ring.MyReceivers.Value),
string.Join(",", state.Ring.NodeRing));
throw;
}
}
}
