Implement basis for Citus Secondaries support (cluster_name). (#616)

Author: DimCitus

src/bin/pg_autoctl/cli_common.c

@@ -72,6 +72,8 @@ static void stop_postgres_and_remove_pgdata_and_config(ConfigFilePaths *pathname
  *		{ "verbose", no_argument, NULL, 'v' },
  *		{ "quiet", no_argument, NULL, 'q' },
  *		{ "help", no_argument, NULL, 'h' },
+ *		{ "secondary", no_argument, NULL, 'z' }
+ *      { "citus-cluster", required_argument, NULL, 'Z' },
  *		{ "candidate-priority", required_argument, NULL, 'P'},
  *		{ "replication-quorum", required_argument, NULL, 'r'},
  *		{ "help", no_argument, NULL, 0 },
@@ -105,11 +107,12 @@ cli_common_keeper_getopts(int argc, char **argv,
 	LocalOptionConfig.prepare_promotion_walreceiver = -1;
 	LocalOptionConfig.postgresql_restart_failure_timeout = -1;
 	LocalOptionConfig.postgresql_restart_failure_max_retries = -1;
-	LocalOptionConfig.pgSetup.settings.candidatePriority =
-		FAILOVER_NODE_CANDIDATE_PRIORITY;
+	LocalOptionConfig.pgSetup.settings.candidatePriority = -1;
 	LocalOptionConfig.pgSetup.settings.replicationQuorum =
 		FAILOVER_NODE_REPLICATION_QUORUM;
 
+	/* default to a "primary" in citus node_role terms */
+	LocalOptionConfig.citusRole = CITUS_ROLE_PRIMARY;
 
 	optind = 0;
 
@@ -182,6 +185,26 @@ cli_common_keeper_getopts(int argc, char **argv,
 				break;
 			}
 
+			case 'z':
+			{
+				/* { "secondary", no_argument, NULL, 'z' } */
+				strlcpy(LocalOptionConfig.citusRoleStr, "secondary", NAMEDATALEN);
+				LocalOptionConfig.citusRole = CITUS_ROLE_SECONDARY;
+				log_trace("--secondary");
+				break;
+			}
+
+			case 'Z':
+			{
+				/* { "citus-cluster", required_argument, NULL, 'Z' }, */
+				strlcpy(LocalOptionConfig.pgSetup.citusClusterName,
+						optarg,
+						NAMEDATALEN);
+				log_trace("--citus-cluster %s",
+						  LocalOptionConfig.pgSetup.citusClusterName);
+				break;
+			}
+
 			case 'U':
 			{
 				/* { "username", required_argument, NULL, 'U' } */
@@ -510,6 +533,41 @@ cli_common_keeper_getopts(int argc, char **argv,
 		exit(EXIT_CODE_BAD_ARGS);
 	}
 
+	/* check --secondary and --candidate-priority */
+	if (LocalOptionConfig.pgSetup.settings.candidatePriority == -1)
+	{
+		/* --candidate-priority has not been used */
+		if (LocalOptionConfig.citusRole == CITUS_ROLE_SECONDARY)
+		{
+			/* a Citus secondary can't be a target for failover */
+			LocalOptionConfig.pgSetup.settings.candidatePriority = 0;
+		}
+		else
+		{
+			/* here we install the default candidate priority */
+			LocalOptionConfig.pgSetup.settings.candidatePriority =
+				FAILOVER_NODE_CANDIDATE_PRIORITY;
+		}
+	}
+	else if (LocalOptionConfig.pgSetup.settings.candidatePriority > 0 &&
+			 LocalOptionConfig.citusRole == CITUS_ROLE_SECONDARY)
+	{
+		log_fatal("Citus does not support secondary roles that are "
+				  "also a candidate for failover: please use --secondary "
+				  "with --candidate-priority 0");
+		exit(EXIT_CODE_BAD_ARGS);
+	}
+
+	/* a --secondary citus worker requires a cluster name */
+	if (LocalOptionConfig.citusRole == CITUS_ROLE_SECONDARY)
+	{
+		if (IS_EMPTY_STRING_BUFFER(LocalOptionConfig.pgSetup.citusClusterName))
+		{
+			log_fatal("When using --secondary, also use --citus-cluster");
+			exit(EXIT_CODE_BAD_ARGS);
+		}
+	}
+
 	if (errors > 0)
 	{
 		commandline_help(stderr);
@@ -573,6 +631,8 @@ cli_common_keeper_getopts(int argc, char **argv,
  *		{ "verbose", no_argument, NULL, 'v' },
  *		{ "quiet", no_argument, NULL, 'q' },
  *		{ "help", no_argument, NULL, 'h' },
+ *		{ "secondary", no_argument, NULL, 'z' },
+ *      { "citus-cluster", required_argument, NULL, 'Z' },
  *		{ "candidate-priority", required_argument, NULL, 'P'},
  *		{ "replication-quorum", required_argument, NULL, 'r'},
  *		{ "help", no_argument, NULL, 0 },
@@ -1253,12 +1313,14 @@ cli_print_version_getopts(int argc, char **argv)
 void
 keeper_cli_print_version(int argc, char **argv)
 {
+	const char *version = PG_AUTOCTL_VERSION;
+
 	if (outputJSON)
 	{
 		JSON_Value *js = json_value_init_object();
 		JSON_Object *root = json_value_get_object(js);
 
-		json_object_set_string(root, "pg_autoctl", PG_AUTOCTL_VERSION);
+		json_object_set_string(root, "pg_autoctl", version);
 		json_object_set_string(root,
 							   "pgautofailover", PG_AUTOCTL_EXTENSION_VERSION);
 		json_object_set_string(root, "pg_major", PG_MAJORVERSION);
@@ -1270,7 +1332,7 @@ keeper_cli_print_version(int argc, char **argv)
 	}
 	else
 	{
-		fformat(stdout, "pg_autoctl version %s\n", PG_AUTOCTL_VERSION);
+		fformat(stdout, "pg_autoctl version %s\n", version);
 		fformat(stdout,
 				"pg_autoctl extension version %s\n",
 				PG_AUTOCTL_EXTENSION_VERSION);
@@ -1333,10 +1395,8 @@ cli_drop_local_node(KeeperConfig *config, bool dropAndDestroy)
 	{
 		if (file_exists(config->pathnames.state))
 		{
-			bool ignoreMonitorErrors = true;
-
 			/* keeper_remove uses log_info() to explain what's happening */
-			if (!keeper_remove(&keeper, config, ignoreMonitorErrors))
+			if (!keeper_remove(&keeper, config))
 			{
 				log_fatal("Failed to remove local node from the pg_auto_failover "
 						  "monitor, see above for details");

src/bin/pg_autoctl/cli_common.h

@@ -109,7 +109,6 @@ extern CommandLine drop_formation_command;
 extern CommandLine perform_failover_command;
 extern CommandLine perform_switchover_command;
 
-
 extern CommandLine *perform_subcommands[];
 extern CommandLine perform_commands;
 

src/bin/pg_autoctl/cli_create_drop_node.c

@@ -300,7 +300,6 @@ cli_create_postgres_getopts(int argc, char **argv)
 		{ "candidate-priority", required_argument, NULL, 'P' },
 		{ "replication-quorum", required_argument, NULL, 'r' },
 		{ "run", no_argument, NULL, 'x' },
-		{ "help", no_argument, NULL, 0 },
 		{ "no-ssl", no_argument, NULL, 'N' },
 		{ "ssl-self-signed", no_argument, NULL, 's' },
 		{ "ssl-mode", required_argument, &ssl_flag, SSL_MODE_FLAG },

src/bin/pg_autoctl/cli_do_root.h

@@ -25,6 +25,9 @@ extern CommandLine do_service_postgres_ctl_commands;
 
 /* src/bin/pg_autoctl/cli_do_show.c */
 extern CommandLine do_show_commands;
+extern CommandLine do_pgsetup_commands;
+extern CommandLine do_service_postgres_ctl_commands;
+extern CommandLine do_service_commands;
 
 /* src/bin/pg_autoctl/cli_do_demo.c */
 extern CommandLine do_demo_commands;
@@ -63,6 +66,8 @@ extern CommandLine do_standby_promote;
 
 extern CommandLine do_discover;
 
+extern CommandLine do_tmux_commands;
+
 /* src/bin/pg_autoctl/cli_do_azure.c */
 extern CommandLine do_azure_ssh;
 

src/bin/pg_autoctl/cli_root.h

@@ -34,6 +34,9 @@ extern CommandLine *create_subcommands[];
 extern CommandLine show_commands;
 extern CommandLine *show_subcommands[];
 
+extern CommandLine show_commands_with_debug;
+extern CommandLine *show_subcommands_with_debug[];
+
 extern CommandLine drop_commands;
 extern CommandLine *drop_subcommands[];
 

src/bin/pg_autoctl/cli_show.c

@@ -55,6 +55,7 @@ static void print_monitor_uri(Monitor *monitor, FILE *stream);
 static void print_formation_uri(SSLOptions *ssl,
 								Monitor *monitor,
 								const char *formation,
+								const char *citusClusterName,
 								FILE *stream);
 static void print_all_uri(SSLOptions *ssl,
 						  Monitor *monitor,
@@ -153,6 +154,7 @@ typedef struct ShowUriOptions
 {
 	bool monitorOnly;
 	char formation[NAMEDATALEN];
+	char citusClusterName[NAMEDATALEN];
 } ShowUriOptions;
 
 static ShowUriOptions showUriOptions = { 0 };
@@ -818,6 +820,7 @@ cli_show_uri_getopts(int argc, char **argv)
 		{ "pgdata", required_argument, NULL, 'D' },
 		{ "monitor", required_argument, NULL, 'm' },
 		{ "formation", required_argument, NULL, 'f' },
+		{ "citus-cluster", required_argument, NULL, 'Z' },
 		{ "json", no_argument, NULL, 'J' },
 		{ "version", no_argument, NULL, 'V' },
 		{ "verbose", no_argument, NULL, 'v' },
@@ -874,6 +877,13 @@ cli_show_uri_getopts(int argc, char **argv)
 				break;
 			}
 
+			case 'Z':
+			{
+				strlcpy(showUriOptions.citusClusterName, optarg, NAMEDATALEN);
+				log_trace("--citus-cluster %s", showUriOptions.citusClusterName);
+				break;
+			}
+
 			case 'V':
 			{
 				/* keeper_cli_print_version prints version and exits. */
@@ -951,8 +961,27 @@ cli_show_uri_getopts(int argc, char **argv)
 		cli_common_get_set_pgdata_or_exit(&(options.pgSetup));
 	}
 
-	/* when --pgdata is given, still initialise our pathnames */
-	if (!IS_EMPTY_STRING_BUFFER(options.pgSetup.pgdata))
+	/*
+	 * When --citus-cluster is used, but not --formation, then we assume
+	 * --formation default
+	 */
+	if (!IS_EMPTY_STRING_BUFFER(showUriOptions.citusClusterName) &&
+		IS_EMPTY_STRING_BUFFER(showUriOptions.formation))
+	{
+		strlcpy(showUriOptions.formation, FORMATION_DEFAULT, NAMEDATALEN);
+	}
+
+	/* use "default" citus cluster name when user didn't provide it */
+	if (IS_EMPTY_STRING_BUFFER(showUriOptions.citusClusterName))
+	{
+		strlcpy(showUriOptions.citusClusterName,
+				DEFAULT_CITUS_CLUSTER_NAME, NAMEDATALEN);
+	}
+
+	cli_common_get_set_pgdata_or_exit(&(options.pgSetup));
+
+	if (!keeper_config_set_pathnames_from_pgdata(&(options.pathnames),
+												 options.pgSetup.pgdata))
 	{
 		if (!keeper_config_set_pathnames_from_pgdata(&(options.pathnames),
 													 options.pgSetup.pgdata))
@@ -1088,8 +1117,10 @@ cli_show_uri(int argc, char **argv)
 	}
 	else if (!IS_EMPTY_STRING_BUFFER(showUriOptions.formation))
 	{
-		(void) print_formation_uri(&ssl, &monitor,
+		(void) print_formation_uri(&ssl,
+								   &monitor,
 								   showUriOptions.formation,
+								   showUriOptions.citusClusterName,
 								   stdout);
 	}
 	else
@@ -1134,12 +1165,14 @@ static void
 print_formation_uri(SSLOptions *ssl,
 					Monitor *monitor,
 					const char *formation,
+					const char *citusClusterName,
 					FILE *stream)
 {
 	char postgresUri[MAXCONNINFO];
 
 	if (!monitor_formation_uri(monitor,
 							   formation,
+							   citusClusterName,
 							   ssl,
 							   postgresUri,
 							   MAXCONNINFO))

src/bin/pg_autoctl/defaults.h

@@ -14,10 +14,10 @@
 #define PG_AUTOCTL_STATE_VERSION 1
 
 /* additional version information for printing version on CLI */
-#define PG_AUTOCTL_VERSION "1.4.2"
+#define PG_AUTOCTL_VERSION "1.5.0"
 
 /* version of the extension that we requite to talk to on the monitor */
-#define PG_AUTOCTL_EXTENSION_VERSION "1.4"
+#define PG_AUTOCTL_EXTENSION_VERSION "1.5.0.1"
 
 /* environment variable to use to make DEBUG facilities available */
 #define PG_AUTOCTL_DEBUG "PG_AUTOCTL_DEBUG"
@@ -85,6 +85,9 @@
 #define POSTGRESQL_FAILS_TO_START_TIMEOUT 20
 #define POSTGRESQL_FAILS_TO_START_RETRIES 3
 
+#define DEFAULT_CITUS_ROLE "primary"
+#define DEFAULT_CITUS_CLUSTER_NAME "default"
+
 #define FAILOVER_FORMATION_NUMBER_SYNC_STANDBYS 1
 #define FAILOVER_NODE_CANDIDATE_PRIORITY 50
 #define FAILOVER_NODE_REPLICATION_QUORUM true

src/bin/pg_autoctl/demoapp.c

@@ -81,7 +81,12 @@ demoapp_grab_formation_uri(DemoAppOptions *options, char *pguri, size_t size,
 	/* allow lots of retries to connect to the monitor at startup */
 	pgsql_set_monitor_interactive_retry_policy(&(monitor.pgsql.retryPolicy));
 
-	if (!monitor_formation_uri(&monitor, options->formation, &ssl, pguri, size))
+	if (!monitor_formation_uri(&monitor,
+							   options->formation,
+							   "default",
+							   &ssl,
+							   pguri,
+							   size))
 	{
 		int groupsCount = 0;
 

src/bin/pg_autoctl/fsm.h

@@ -69,6 +69,11 @@ bool fsm_prepare_cascade(Keeper *keeper);
 bool fsm_follow_new_primary(Keeper *keeper);
 bool fsm_cleanup_and_resume_as_primary(Keeper *keeper);
 
+/*
+ * Extra helpers.
+ */
+bool prepare_replication(Keeper *keeper, NodeState otherNodeState);
+
 /*
  * Generic API to use the previous definitions.
  */

src/bin/pg_autoctl/keeper.c

@@ -1310,6 +1310,7 @@ keeper_register_and_init(Keeper *keeper, NodeState initialState)
 								  config->pgSetup.pgKind,
 								  config->pgSetup.settings.candidatePriority,
 								  config->pgSetup.settings.replicationQuorum,
+								  config->pgSetup.citusClusterName,
 								  &mayRetry,
 								  &assignedState))
 		{
@@ -1439,7 +1440,7 @@ keeper_register_and_init(Keeper *keeper, NodeState initialState)
  * local state file.
  */
 bool
-keeper_remove(Keeper *keeper, KeeperConfig *config, bool ignore_monitor_errors)
+keeper_remove(Keeper *keeper, KeeperConfig *config)
 {
 	int errors = 0;
 
@@ -1473,13 +1474,8 @@ keeper_remove(Keeper *keeper, KeeperConfig *config, bool ignore_monitor_errors)
 							config->hostname,
 							config->pgSetup.pgport))
 		{
-			/* we already logged about errors */
-			errors++;
-
-			if (!ignore_monitor_errors)
-			{
-				return false;
-			}
+			/* errors have already been logged */
+			return false;
 		}
 	}