Review pg_autoctl show state output, and docs. (#617)

* Review the FSM diagram.

Make it obvious that we have more than one state where the application can
successfully connect to a read-write node.

* Review pg_autoctl show state output.

Make it clear which kind of connection is given by which node, in particular
that when in the SINGLE or WAIT_PRIMARY state we are still delivering a
read-write connection to the application.

The "Reachable" column is now a "Connection" column and the node health as
seen by the monitor is encoded in the single-char *, !, or ?, depending on
the health value (1, 0, -1).

* Per review, simplify indications when it's all good.

Author: DimCitus

docs/failover-state-machine.rst

@@ -65,6 +65,11 @@ all the other nodes.
 
    pg_auto_failover Finite State Machine diagram
 
+In the previous diagram we can see that we have a list of six states where
+the application can connect to a read-write Postgres service: ``single``,
+``wait_primary``, ``primary``, ``prepare_maintenance``, ``apply_settings``,
+and ``join_primary``.
+
 Init
 ^^^^
 

docs/fsm.png

@@ -39,11 +39,11 @@
    nodes={text height=.7em, text depth=.2em,
      draw=black!20, thick, fill=blue!20, font=\footnotesize}]
         {
-          init [circle,draw,x=-10,y=10,color=white,fill=black!70] -> single [circle,draw=black,color=white,fill=black!50];
+          init [circle,draw,x=-10,y=10,color=white,fill=black!70] -> single [font=\normalsize, draw=black, circle];
           init -> "wait standby" [fill=violet!30];
-          single -> "wait primary" ;
-          "wait primary" -> primary [circle,draw=black,circle];
-          "join primary" -> primary;
+          single -> "wait primary" [font=\normalsize, draw=black];
+          "wait primary" -> primary [font=\normalsize, draw=black, circle];
+          "join primary" [font=\normalsize, draw=black] -> primary;
           "wait standby" -> "catching up" [fill=violet!30];
           "catching up" -> secondary [circle,draw=black,circle,fill=violet!30];
           primary -> "join primary" ;
@@ -59,14 +59,14 @@
           draining -> "demote timeout" ;
           "demote timeout" -> demoted ;
 
-          primary -> "apply settings" [fill=green!20];
+          primary -> "apply settings" [font=\normalsize, draw=black];
           "apply settings" -> primary ;
           "apply settings" -> draining ;
           "apply settings" -> demoted ;
           "apply settings" -> "demote timeout" ;
           "apply settings" -> primary ;
 
-           primary -> "prepare maintenance" [fill=blue!10];
+           primary -> "prepare maintenance" [font=\normalsize, draw=black];
            "prepare maintenance" -> maintenance [fill=black!20];
            secondary -> "wait maintenance" [fill=violet!20];
            "catching up" -> "wait maintenance" ;

docs/tikz/fsm.svg

@@ -30,7 +30,7 @@ nodestatePrepareHeaders(CurrentNodeStateArray *nodesArray,
 	nodesArray->headers.maxNodeSize = 5;  /* "Node" */
 	nodesArray->headers.maxLSNSize = 3;   /* "LSN" */
 	nodesArray->headers.maxStateSize = MAX_NODE_STATE_LEN;
-	nodesArray->headers.maxHealthSize = strlen("Reachable");
+	nodesArray->headers.maxHealthSize = strlen("read-write *");
 
 	/*
 	 * Dynamically adjust our display output to the length of the longer
@@ -184,11 +184,11 @@ nodestateAdjustHeaders(NodeAddressHeaders *headers,
 	if (headers->maxHealthSize == 0)
 	{
 		/*
-		 * Reachable, which is longer than "unknown", "yes", and "no", so
-		 * that's all we use here, a static length actually. Which is good,
-		 * because a NodeAddress does not know its own health anyway.
+		 * Connection is one of "read-only", "read-write", or "unknown",
+		 * followed by a mark for the health check (*, !, or ?), so we need as
+		 * much space as the full sample "read-write *":
 		 */
-		headers->maxHealthSize = strlen("Reachable");
+		headers->maxHealthSize = strlen("read-write *");
 	}
 
 	if (nameLen > headers->maxNameSize)
@@ -224,7 +224,7 @@ nodestatePrintHeader(NodeAddressHeaders *headers)
 			headers->maxNodeSize, "Node",
 			headers->maxHostSize, "Host:Port",
 			headers->maxLSNSize, "LSN",
-			headers->maxHealthSize, "Reachable",
+			headers->maxHealthSize, "Connection",
 			headers->maxStateSize, "Current State",
 			headers->maxStateSize, "Assigned State");
 
@@ -249,19 +249,31 @@ nodestatePrintNodeState(NodeAddressHeaders *headers,
 {
 	char hostport[BUFSIZE] = { 0 };
 	char composedId[BUFSIZE] = { 0 };
+	char connection[BUFSIZE] = { 0 };
+	char healthChar = nodestateHealthToChar(nodeState->health);
 
 	(void) nodestatePrepareNode(headers,
 								&(nodeState->node),
 								nodeState->groupId,
 								hostport,
 								composedId);
 
+	if (healthChar == ' ')
+	{
+		sformat(connection, BUFSIZE, "%s", nodestateConnectionType(nodeState));
+	}
+	else
+	{
+		sformat(connection, BUFSIZE, "%s %c",
+				nodestateConnectionType(nodeState), healthChar);
+	}
+
 	fformat(stdout, "%*s | %*s | %*s | %*s | %*s | %*s | %*s\n",
 			headers->maxNameSize, nodeState->node.name,
 			headers->maxNodeSize, composedId,
 			headers->maxHostSize, hostport,
 			headers->maxLSNSize, nodeState->node.lsn,
-			headers->maxHealthSize, nodestateHealthToString(nodeState->health),
+			headers->maxHealthSize, connection,
 			headers->maxStateSize, NodeStateToString(nodeState->reportedState),
 			headers->maxStateSize, NodeStateToString(nodeState->goalState));
 }
@@ -346,6 +358,9 @@ nodestateAsJSON(CurrentNodeState *nodeState, JSON_Value *js)
 	json_object_set_string(jsobj, "reachable",
 						   nodestateHealthToString(nodeState->health));
 
+	json_object_set_string(jsobj, "conntype",
+						   nodestateConnectionType(nodeState));
+
 	return true;
 }
 
@@ -382,6 +397,92 @@ nodestateHealthToString(int health)
 }
 
 
+/*
+ * Transform the health column from a monitor into a single char.
+ */
+char
+nodestateHealthToChar(int health)
+{
+	switch (health)
+	{
+		case -1:
+		{
+			return '?';
+		}
+
+		case 0:
+		{
+			return '!';
+		}
+
+		case 1:
+		{
+			return ' ';
+		}
+
+		default:
+		{
+			log_error("BUG in nodestateHealthToString: health = %d", health);
+			return '-';
+		}
+	}
+}
+
+
+/*
+ * nodestateConnectionType returns one of "read-write" or "read-only".
+ */
+char *
+nodestateConnectionType(CurrentNodeState *nodeState)
+{
+	switch (nodeState->reportedState)
+	{
+		case SINGLE_STATE:
+		case PRIMARY_STATE:
+		case WAIT_PRIMARY_STATE:
+		case JOIN_PRIMARY_STATE:
+		case PREPARE_MAINTENANCE_STATE:
+		case APPLY_SETTINGS_STATE:
+		{
+			return "read-write";
+		}
+
+		case SECONDARY_STATE:
+		case CATCHINGUP_STATE:
+		case PREP_PROMOTION_STATE:
+		case STOP_REPLICATION_STATE:
+		case WAIT_MAINTENANCE_STATE:
+		case FAST_FORWARD_STATE:
+		case JOIN_SECONDARY_STATE:
+		{
+			return "read-only";
+		}
+
+		/* in those states Postgres is known to be stopped/down */
+		case NO_STATE:
+		case INIT_STATE:
+		case WAIT_STANDBY_STATE:
+		case DEMOTED_STATE:
+		case DEMOTE_TIMEOUT_STATE:
+		case DRAINING_STATE:
+		case MAINTENANCE_STATE:
+		case REPORT_LSN_STATE:
+		{
+			return "none";
+		}
+
+		case ANY_STATE:
+		{
+			return "unknown";
+		}
+
+			/* default: is intentionally left out to have compiler check */
+	}
+
+	return "unknown";
+}
+
+
 /*
  * nodestate_log logs a CurrentNodeState, usually that comes from a
  * notification message we parse.

docs/tikz/fsm.tex

@@ -89,6 +89,8 @@ void prepareHostNameSeparator(char nameSeparatorHeader[], int size);
 bool nodestateAsJSON(CurrentNodeState *nodeState, JSON_Value *js);
 
 char * nodestateHealthToString(int health);
+char nodestateHealthToChar(int health);
+char * nodestateConnectionType(CurrentNodeState *nodeState);
 void nodestate_log(CurrentNodeState *nodeState, int logLevel, int nodeId);
 
 void printNodeArray(NodeAddressArray *nodesArray);