Attempt at fixing an often failing test. (#696)

When trying to find if a candidate has already been selected, avoid choosing
a node that is in WAIT_PRIMARY. As we don't change the state of such a node
when it becomes unhealthy, that might lead to strange results when two
failover situations happen one after another.

Also guard demoted to cachingup transitions to only happen when the current
primary is known healthy.

Author: DimCitus

src/monitor/group_state_machine.c

@@ -633,6 +633,7 @@ ProceedGroupState(AutoFailoverNode *activeNode)
 	 * concurrently making progress.
 	 */
 	if (IsCurrentState(activeNode, REPLICATION_STATE_DEMOTED) &&
+		IsHealthy(primaryNode) &&
 		((primaryNode->reportedState == REPLICATION_STATE_WAIT_PRIMARY ||
 		  primaryNode->reportedState == REPLICATION_STATE_JOIN_PRIMARY) &&
 		 primaryNode->goalState == REPLICATION_STATE_PRIMARY))
@@ -658,6 +659,7 @@ ProceedGroupState(AutoFailoverNode *activeNode)
 	 *  demoted -> catchingup
 	 */
 	if (IsCurrentState(activeNode, REPLICATION_STATE_DEMOTED) &&
+		IsHealthy(primaryNode) &&
 		(IsCurrentState(primaryNode, REPLICATION_STATE_JOIN_PRIMARY) ||
 		 IsCurrentState(primaryNode, REPLICATION_STATE_WAIT_PRIMARY) ||
 		 IsCurrentState(primaryNode, REPLICATION_STATE_PRIMARY)))

src/monitor/node_metadata.c

@@ -1565,11 +1565,7 @@ IsBeingPromoted(AutoFailoverNode *node)
 
 			(node->reportedState == REPLICATION_STATE_STOP_REPLICATION &&
 			 (node->goalState == REPLICATION_STATE_STOP_REPLICATION ||
-			  node->goalState == REPLICATION_STATE_WAIT_PRIMARY)) ||
-
-			(node->reportedState == REPLICATION_STATE_WAIT_PRIMARY &&
-			 (node->goalState == REPLICATION_STATE_WAIT_PRIMARY ||
-			  node->goalState == REPLICATION_STATE_PRIMARY)));
+			  node->goalState == REPLICATION_STATE_WAIT_PRIMARY)));
 }