public void MarkNodeAsUNREACHABLEWhenWePullTheNetwork()
{
// let them send at least one heartbeat to each other after the gossip convergence
// because for new joining nodes we remove them from the failure detector when
// receive gossip
Thread.Sleep(Dilated(TimeSpan.FromSeconds(2)));
RunOn(() =>
{
// pull network for victim node from all nodes
AllBut(_victim.Value).ForEach(role =>
{
TestConductor.Blackhole(_victim.Value, role, ThrottleTransportAdapter.Direction.Both).Wait();
});
}, _config.First);
EnterBarrier("unplug_victim");
var allButVictim = AllBut(_victim.Value).ToArray();
RunOn(() =>
{
var victimAddress = GetAddress(_victim.Value);
allButVictim.ForEach(name => MarkNodeAsUnavailable(GetAddress(name)));
Within(TimeSpan.FromSeconds(30), () =>
{
// victim becomes all alone
AwaitAssert(() =>
{
var members = ClusterView.Members; // to snapshot the object
Assert.Equal(Roles.Count - 1, ClusterView.UnreachableMembers.Count);
});
var addresses = allButVictim.Select(GetAddress).ToList();
Assert.True(ClusterView.UnreachableMembers.Select(x => x.Address).All(y => addresses.Contains(y)));
});
}, _victim.Value);
RunOn(() =>
{
MarkNodeAsUnavailable(GetAddress(_victim.Value));
Within(TimeSpan.FromSeconds(30), () =>
{
// victim becomes unreachable
AwaitAssert(() =>
{
var members = ClusterView.Members; // to snapshot the object
Assert.Equal(1, ClusterView.UnreachableMembers.Count);
});
AwaitSeenSameState(allButVictim.Select(GetAddress).ToArray());
// still once unreachable
Assert.Equal(1, ClusterView.UnreachableMembers.Count);
Assert.Equal(Node(_victim.Value).Address, ClusterView.UnreachableMembers.First().Address);
Assert.Equal(MemberStatus.Up, ClusterView.UnreachableMembers.First().Status);
});
}, allButVictim);
EndBarrier();
}