Implement pg_autoctl create postgres --pg-hba-lan option. (#561)

* Implement pg_autoctl create postgres --pg-hba-lan option.

With this option pg_autoctl edits the pg_hba.conf for Postgres to grant
connection privileges on the detected LAN for the --dbname database and for
the --username user.

The LAN detection is done the same way as with the monitor.

* Implement a retry loop when trying to figure local IP address.

We connect to the monitor to figure out the local hostname and IP address,
and the monitor might not be running yet. In that case, we might want to
persist with some retries before failing back to the user.

* Install HBA on the monitor early.

This avoids some retry attemps from the other nodes at first startup.

* When we WARN about hostname, use IP address in HBA.

Rather than adding an hostname that we know faulty in the HBA file, we add
one of the IP addresses of the hostname instead. We might want to revisit
this (add all IP addresses maybe? or find the one we want to add by
connecting, like we do for the monitor?), but it allows the docker-compose
demo to just work with a minimum of trouble.

* Add a docker-compose.yml setup.

This allows to run a demo where the monitor and the Postgres nodes are each
running in their own container and connecting through the docker provided
TCP/IP network.

Author: DimCitus

docker-compose.yml

@@ -0,0 +1,60 @@
+version: "3.9"  # optional since v1.27.0
+services:
+  monitor:
+    image: citusdata/pg_auto_failover:demo
+    environment:
+      PGDATA: /tmp/pgaf
+      PG_AUTOCTL_DEBUG: 1
+    command: pg_autoctl create monitor --ssl-self-signed --auth trust --run
+    expose:
+      - 5432
+  node1:
+    image: citusdata/pg_auto_failover:demo
+    environment:
+      PGDATA: /tmp/pgaf
+      PG_AUTOCTL_DEBUG: 1
+    command: [
+    "pg_autoctl", "create", "postgres",
+    "--ssl-self-signed",
+    "--auth", "trust",
+    "--pg-hba-lan",
+    "--username", "ad",
+    "--dbname", "analytics",
+    "--monitor", "postgresql://autoctl_node@monitor/pg_auto_failover",
+    "--run"]
+    expose:
+      - 5432
+  node2:
+    image: citusdata/pg_auto_failover:demo
+    expose:
+      - 5432
+    environment:
+      PGDATA: /tmp/pgaf
+      PG_AUTOCTL_DEBUG: 1
+    command: [
+    "pg_autoctl", "create", "postgres",
+    "--ssl-self-signed",
+    "--auth", "trust",
+    "--pg-hba-lan",
+    "--username", "ad",
+    "--dbname", "analytics",
+    "--monitor", "postgresql://autoctl_node@monitor/pg_auto_failover",
+    "--run"]
+    expose:
+      - 5432
+  node3:
+    image: citusdata/pg_auto_failover:demo
+    environment:
+      PGDATA: /tmp/pgaf
+      PG_AUTOCTL_DEBUG: 1
+    command: [
+    "pg_autoctl", "create", "postgres",
+    "--ssl-self-signed",
+    "--auth", "trust",
+    "--pg-hba-lan",
+    "--username", "ad",
+    "--dbname", "analytics",
+    "--monitor", "postgresql://autoctl_node@monitor/pg_auto_failover",
+    "--run"]
+    expose:
+      - 5432

docs/ref/reference.rst

@@ -424,6 +424,7 @@ The other commands accept the same set of options.
     --monitor         pg_auto_failover Monitor Postgres URL
     --auth            authentication method for connections from monitor
     --skip-pg-hba     skip editing pg_hba.conf rules
+    --pg-hba-lan      edit pg_hba.conf rules for --dbname in detected LAN
     --candidate-priority    priority of the node to be promoted to become primary
     --replication-quorum    true if node participates in write quorum
     --ssl-self-signed setup network encryption using self signed certificates (does NOT protect against MITM)

src/bin/pg_autoctl/cli_common.c

@@ -35,6 +35,7 @@
 KeeperConfig keeperOptions;
 bool createAndRun = false;
 bool outputJSON = false;
+bool openAppHBAonLAN = false;
 int ssl_flag = 0;
 
 /* stores --node-id, only used with --disable-monitor */
@@ -57,7 +58,8 @@ static void stop_postgres_and_remove_pgdata_and_config(ConfigFilePaths *pathname
  *		{ "proxyport", required_argument, NULL, 'y' },
  *		{ "username", required_argument, NULL, 'U' },
  *		{ "auth", required_argument, NULL, 'A' },
- *		{ "skip-pg-hba", required_argument, NULL, 'S' },
+ *		{ "skip-pg-hba", no_argument, NULL, 'S' },
+ *		{ "pg-hba-lan", no_argument, NULL, 'L' },
  *		{ "dbname", required_argument, NULL, 'd' },
  *		{ "name", required_argument, NULL, 'a' },
  *		{ "hostname", required_argument, NULL, 'n' },
@@ -211,13 +213,41 @@ cli_common_keeper_getopts(int argc, char **argv,
 					log_error("Please use either --auth or --skip-pg-hba");
 				}
 
+				/* force default authentication method then */
 				strlcpy(LocalOptionConfig.pgSetup.authMethod,
-						SKIP_HBA_AUTH_METHOD,
+						DEFAULT_AUTH_METHOD,
 						NAMEDATALEN);
+
+				strlcpy(LocalOptionConfig.pgSetup.hbaLevelStr,
+						pgsetup_hba_level_to_string(HBA_EDIT_SKIP),
+						sizeof(LocalOptionConfig.pgSetup.hbaLevelStr));
+
+				LocalOptionConfig.pgSetup.hbaLevel = HBA_EDIT_SKIP;
+
 				log_trace("--skip-pg-hba");
 				break;
 			}
 
+			case 'L':
+			{
+				/* { "pg-hba-lan", required_argument, NULL, 'L' }, */
+				if (LocalOptionConfig.pgSetup.hbaLevel != HBA_EDIT_UNKNOWN &&
+					LocalOptionConfig.pgSetup.hbaLevel != HBA_EDIT_LAN)
+				{
+					errors++;
+					log_error("Please use either --skip-pg-hba or --pg-hba-lan");
+				}
+
+				strlcpy(LocalOptionConfig.pgSetup.hbaLevelStr,
+						pgsetup_hba_level_to_string(HBA_EDIT_LAN),
+						sizeof(LocalOptionConfig.pgSetup.hbaLevelStr));
+
+				LocalOptionConfig.pgSetup.hbaLevel = HBA_EDIT_LAN;
+
+				log_trace("--pg-hba-lan");
+				break;
+			}
+
 			case 'd':
 			{
 				/* { "dbname", required_argument, NULL, 'd' } */
@@ -478,6 +508,12 @@ cli_common_keeper_getopts(int argc, char **argv,
 		exit(EXIT_CODE_BAD_ARGS);
 	}
 
+	/* the default HBA editing level is MINIMAL, time to install it */
+	if (LocalOptionConfig.pgSetup.hbaLevel == HBA_EDIT_UNKNOWN)
+	{
+		LocalOptionConfig.pgSetup.hbaLevel = HBA_EDIT_MINIMAL;
+	}
+
 	/*
 	 * Now, all commands need PGDATA validation.
 	 */
@@ -513,7 +549,8 @@ cli_common_keeper_getopts(int argc, char **argv,
  *		{ "proxyport", required_argument, NULL, 'y' },
  *		{ "username", required_argument, NULL, 'U' },
  *		{ "auth", required_argument, NULL, 'A' },
- *		{ "skip-pg-hba", required_argument, NULL, 'S' },
+ *		{ "skip-pg-hba", no_argument, NULL, 'S' },
+ *		{ "pg-hba-lan", no_argument, NULL, 'L' },
  *		{ "dbname", required_argument, NULL, 'd' },
  *		{ "hostname", required_argument, NULL, 'n' },
  *		{ "formation", required_argument, NULL, 'f' },

src/bin/pg_autoctl/cli_common.h

@@ -22,6 +22,7 @@ extern MonitorConfig monitorOptions;
 extern KeeperConfig keeperOptions;
 extern bool createAndRun;
 extern bool outputJSON;
+extern bool openAppHBAonLAN;
 
 #define SSL_CA_FILE_FLAG 1      /* root public certificate */
 #define SSL_CRL_FILE_FLAG 2     /* certificates revocation list */

src/bin/pg_autoctl/cli_create_drop_node.c

@@ -97,9 +97,10 @@ CommandLine create_postgres_command =
 		"  --monitor         pg_auto_failover Monitor Postgres URL\n"
 		"  --auth            authentication method for connections from monitor\n"
 		"  --skip-pg-hba     skip editing pg_hba.conf rules\n"
+		"  --pg-hba-lan      edit pg_hba.conf rules for --dbname in detected LAN\n"
+		KEEPER_CLI_SSL_OPTIONS
 		"  --candidate-priority    priority of the node to be promoted to become primary\n"
-		"  --replication-quorum    true if node participates in write quorum\n"
-		KEEPER_CLI_SSL_OPTIONS,
+		"  --replication-quorum    true if node participates in write quorum\n",
 		cli_create_postgres_getopts,
 		cli_create_postgres);
 
@@ -284,6 +285,7 @@ cli_create_postgres_getopts(int argc, char **argv)
 		{ "username", required_argument, NULL, 'U' },
 		{ "auth", required_argument, NULL, 'A' },
 		{ "skip-pg-hba", no_argument, NULL, 'S' },
+		{ "pg-hba-lan", no_argument, NULL, 'L' },
 		{ "dbname", required_argument, NULL, 'd' },
 		{ "name", required_argument, NULL, 'a' },
 		{ "hostname", required_argument, NULL, 'n' },
@@ -311,7 +313,7 @@ cli_create_postgres_getopts(int argc, char **argv)
 
 	int optind =
 		cli_create_node_getopts(argc, argv, long_options,
-								"C:D:H:p:l:U:A:Sd:a:n:f:m:MI:RVvqhP:r:xsN",
+								"C:D:H:p:l:U:A:SLd:a:n:f:m:MI:RVvqhP:r:xsN",
 								&options);
 
 	/* publish our option parsing in the global variable */
@@ -470,9 +472,13 @@ cli_create_monitor_getopts(int argc, char **argv)
 					log_error("Please use either --auth or --skip-pg-hba");
 				}
 
+				/* force default authentication method then */
 				strlcpy(options.pgSetup.authMethod,
-						SKIP_HBA_AUTH_METHOD,
+						DEFAULT_AUTH_METHOD,
 						NAMEDATALEN);
+
+				options.pgSetup.hbaLevel = HBA_EDIT_SKIP;
+
 				log_trace("--skip-pg-hba");
 				break;
 			}
@@ -1291,12 +1297,40 @@ discover_hostname(char *hostname, int size,
 	char localIpAddr[BUFSIZE];
 	char hostnameCandidate[_POSIX_HOST_NAME_MAX];
 
-	/* fetch our local address among the network interfaces */
-	if (!fetchLocalIPAddress(ipAddr, BUFSIZE, monitorHostname, monitorPort))
+	ConnectionRetryPolicy retryPolicy = { 0 };
+
+	/* retry connecting to the monitor when it's not available */
+	(void) pgsql_set_monitor_interactive_retry_policy(&retryPolicy);
+
+	while (!pgsql_retry_policy_expired(&retryPolicy))
 	{
-		log_fatal("Failed to find a local IP address, "
-				  "please provide --hostname.");
-		return false;
+		bool mayRetry = false;
+
+		/* fetch our local address among the network interfaces */
+		if (fetchLocalIPAddress(ipAddr, BUFSIZE, monitorHostname, monitorPort,
+								LOG_DEBUG, &mayRetry))
+		{
+			/* success: break out of the retry loop */
+			break;
+		}
+
+		if (!mayRetry)
+		{
+			log_fatal("Failed to find a local IP address, "
+					  "please provide --hostname.");
+			return false;
+		}
+
+		int sleepTimeMs =
+			pgsql_compute_connection_retry_sleep_time(&retryPolicy);
+
+		log_warn("Failed to connect to \"%s\" on port %d "
+				 "to discover this machine hostname, "
+				 "retrying in %d ms.",
+				 monitorHostname, monitorPort, sleepTimeMs);
+
+		/* we have milliseconds, pg_usleep() wants microseconds */
+		(void) pg_usleep(sleepTimeMs * 1000);
 	}
 
 	/* from there on we can take the ipAddr as the default --hostname */
@@ -1364,6 +1398,7 @@ check_hostname(const char *hostname)
 	else
 	{
 		char cidr[BUFSIZE];
+		char ipaddr[BUFSIZE];
 
 		if (!fetchLocalCIDR(hostname, cidr, BUFSIZE))
 		{
@@ -1373,6 +1408,6 @@ check_hostname(const char *hostname)
 		}
 
 		/* use pghba_check_hostname for log diagnostics */
-		(void) pghba_check_hostname(hostname);
+		(void) pghba_check_hostname(hostname, ipaddr, sizeof(ipaddr));
 	}
 }

src/bin/pg_autoctl/cli_do_misc.c

@@ -188,6 +188,7 @@ keeper_cli_create_monitor_user(int argc, char **argv)
 									  PG_AUTOCTL_HEALTH_PASSWORD,
 									  monitorHostname,
 									  "trust",
+									  HBA_EDIT_MINIMAL,
 									  connlimit))
 	{
 		log_fatal("Failed to create the database user that the pg_auto_failover "

src/bin/pg_autoctl/cli_do_root.c

@@ -572,7 +572,7 @@ keeper_cli_keeper_setup_getopts(int argc, char **argv)
 
 	int optind = cli_common_keeper_getopts(argc, argv,
 										   long_options,
-										   "C:D:H:p:l:U:A:Sd:n:f:m:MRVvqhP:r:xsN",
+										   "C:D:H:p:l:U:A:SLd:n:f:m:MRVvqhP:r:xsN",
 										   &options,
 										   &sslCommandLineOptions);
 

src/bin/pg_autoctl/cli_do_show.c

@@ -88,10 +88,12 @@ static void
 cli_show_ipaddr(int argc, char **argv)
 {
 	char ipAddr[BUFSIZE];
+	bool mayRetry = false;
 
 	if (!fetchLocalIPAddress(ipAddr, BUFSIZE,
 							 DEFAULT_INTERFACE_LOOKUP_SERVICE_NAME,
-							 DEFAULT_INTERFACE_LOOKUP_SERVICE_PORT))
+							 DEFAULT_INTERFACE_LOOKUP_SERVICE_PORT,
+							 LOG_WARN, &mayRetry))
 	{
 		log_warn("Failed to determine network configuration.");
 		exit(EXIT_CODE_INTERNAL_ERROR);
@@ -111,10 +113,12 @@ cli_show_cidr(int argc, char **argv)
 {
 	char ipAddr[BUFSIZE];
 	char cidr[BUFSIZE];
+	bool mayRetry = false;
 
 	if (!fetchLocalIPAddress(ipAddr, BUFSIZE,
 							 DEFAULT_INTERFACE_LOOKUP_SERVICE_NAME,
-							 DEFAULT_INTERFACE_LOOKUP_SERVICE_PORT))
+							 DEFAULT_INTERFACE_LOOKUP_SERVICE_PORT,
+							 LOG_WARN, &mayRetry))
 	{
 		log_warn("Failed to determine network configuration.");
 		exit(EXIT_CODE_INTERNAL_ERROR);
@@ -207,6 +211,8 @@ cli_show_hostname(int argc, char **argv)
 	char monitorHostname[_POSIX_HOST_NAME_MAX];
 	int monitorPort = pgsetup_get_pgport();
 
+	bool mayRetry = false;
+
 	/*
 	 * When no argument is used, use hostname(3) and 5432, as we would for a
 	 * monitor (pg_autoctl create monitor).
@@ -256,7 +262,9 @@ cli_show_hostname(int argc, char **argv)
 	}
 
 	/* fetch the default local address used when connecting remotely */
-	if (!fetchLocalIPAddress(ipAddr, BUFSIZE, monitorHostname, monitorPort))
+	if (!fetchLocalIPAddress(ipAddr, BUFSIZE,
+							 monitorHostname, monitorPort,
+							 LOG_WARN, &mayRetry))
 	{
 		log_warn("Failed to determine network configuration.");
 		exit(EXIT_CODE_INTERNAL_ERROR);
@@ -321,6 +329,7 @@ cli_show_reverse(int argc, char **argv)
 		log_fatal("Failed to find an IP address for hostname \"%s\" that "
 				  "matches hostname again in a reverse-DNS lookup.",
 				  hostname);
+		log_info("Continuing with IP address \"%s\"", ipaddr);
 		exit(EXIT_CODE_INTERNAL_ERROR);
 	}
 

src/bin/pg_autoctl/cli_do_tmux.c

@@ -646,7 +646,10 @@ tmux_pg_autoctl_create_postgres(PQExpBuffer script,
 
 	tmux_add_send_keys_command(script,
 							   "%s create postgres %s "
-							   "--monitor %s --name %s "
+							   "--monitor %s "
+							   "--name %s "
+							   "--dbname demo "
+							   "--pg-hba-lan "
 							   "--replication-quorum %s "
 							   "--candidate-priority %d "
 							   "--run",

src/bin/pg_autoctl/defaults.h

@@ -32,7 +32,6 @@
 #define DEFAULT_DATABASE_NAME "postgres"
 #define DEFAULT_USERNAME "postgres"
 #define DEFAULT_AUTH_METHOD "trust"
-#define SKIP_HBA_AUTH_METHOD "skip"
 #define REPLICATION_SLOT_NAME_DEFAULT "pgautofailover_standby"
 #define REPLICATION_SLOT_NAME_PATTERN "^pgautofailover_standby_"
 #define REPLICATION_PASSWORD_DEFAULT NULL