Implement online enable/disable monitor support. (#591)

This allows for adjusting to a lost monitor while keeping our services
online. At run-time, we can now

  $ pg_autoctl disable monitor
  $ pg_autoctl enable monitor --monitor ...

Taking care of the order of operations, the cluster can continue behaving
correctly with a minimum amount of disturbance.

Author: DimCitus

.travis.yml

@@ -14,14 +14,22 @@ env:
 matrix:
   fast_finish: true
   include:
-    - env: PGVERSION=10 TEST=single
-    - env: PGVERSION=11 TEST=single
-    - env: PGVERSION=12 TEST=single
-    - env: PGVERSION=13 TEST=single
     - env: PGVERSION=10 TEST=multi
     - env: PGVERSION=11 TEST=multi
     - env: PGVERSION=12 TEST=multi
     - env: PGVERSION=13 TEST=multi
+    - env: PGVERSION=10 TEST=single
+    - env: PGVERSION=11 TEST=single
+    - env: PGVERSION=12 TEST=single
+    - env: PGVERSION=13 TEST=single
+    - env: PGVERSION=10 TEST=ssl
+    - env: PGVERSION=11 TEST=ssl
+    - env: PGVERSION=12 TEST=ssl
+    - env: PGVERSION=13 TEST=ssl
+    - env: PGVERSION=10 TEST=monitor
+    - env: PGVERSION=11 TEST=monitor
+    - env: PGVERSION=12 TEST=monitor
+    - env: PGVERSION=13 TEST=monitor
     - env: LINTING=true
 before_install:
   - git clone -b v0.7.18 --depth 1 https://github.com/citusdata/tools.git

Makefile

@@ -7,18 +7,45 @@ DOCKER_RUN_OPTS = --privileged  -ti --rm
 
 NOSETESTS = $(shell which nosetests3 || which nosetests)
 
+# Tests for the monitor
+TESTS_MONITOR  = test_extension_update
+TESTS_MONITOR += test_installcheck
+TESTS_MONITOR += test_monitor_disabled
+TESTS_MONITOR += test_replace_monitor
+
+# Tests for single standby
+TESTS_SINGLE  = test_auth
+TESTS_SINGLE += test_basic_operation
+TESTS_SINGLE += test_basic_operation_listen_flag
+TESTS_SINGLE += test_create_run
+TESTS_SINGLE += test_create_standby_with_pgdata
+TESTS_SINGLE += test_debian_clusters
+TESTS_SINGLE += test_ensure
+TESTS_SINGLE += test_skip_pg_hba
+
+# Tests for SSL
+TESTS_SSL  = test_enable_ssl
+TESTS_SSL += test_ssl_cert
+TESTS_SSL += test_ssl_self_signed
+
 # Tests for multiple standbys
-MULTI_SB_TESTS  = $(basename $(notdir $(wildcard tests/test*_multi*)))
-MULTI_SB_TESTS += $(basename $(notdir $(wildcard tests/test*_disabled*)))
+TESTS_MULTI  = test_multi_async
+TESTS_MULTI += test_multi_ifdown
+TESTS_MULTI += test_multi_maintenance
+TESTS_MULTI += test_multi_standbys
 
 # TEST indicates the testfile to run
 TEST ?=
 ifeq ($(TEST),)
 	TEST_ARGUMENT = --where=tests
 else ifeq ($(TEST),multi)
-	TEST_ARGUMENT = --where=tests --tests=$(MULTI_SB_TESTS)
+	TEST_ARGUMENT = --where=tests --tests=$(TESTS_MULTI)
 else ifeq ($(TEST),single)
-	TEST_ARGUMENT = --where=tests --exclude='_multi_' --exclude='disabled'
+	TEST_ARGUMENT = --where=tests --tests=$(TESTS_SINGLE)
+else ifeq ($(TEST),monitor)
+	TEST_ARGUMENT = --where=tests --tests=$(TESTS_MONITOR)
+else ifeq ($(TEST),ssl)
+	TEST_ARGUMENT = --where=tests --tests=$(TESTS_SSL)
 else
 	TEST_ARGUMENT = $(TEST:%=tests/%.py)
 endif

docs/conf.py

@@ -71,9 +71,9 @@ def __init__(self, **options):
 # built documents.
 #
 # The short X.Y version.
-version = "1.4"
+version = "1.5"
 # The full version, including alpha/beta/rc tags.
-release = "1.4.2"
+release = "1.5.0.2"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/faq.rst

@@ -165,26 +165,10 @@ node metadata. Specifically we need the nodes state to be in sync with what
 each ``pg_autoctl`` process has received the last time they could contact
 the monitor, before it has been unavailable.
 
-Barring that, the way forward is to register your nodes again to the new
-monitor. To be able to register again, we need to have a clean initial local
-state on every node, and the ``pg_autoctl drop node`` command achieves that.
+It is possible to register nodes that are currently running to a new monitor
+without restarting Postgres on the primary. For that, the procedure
+mentionned in :ref:`replacing_monitor_online` must be followed, using the
+following commands::
 
-.. warning::
-
-   This procedure includes a step where the Postgres service has to be
-   stopped and started again.
-
-On every Postgres node, starting with the current primary, remove the local node state and register the node
-again to the new running monitor::
-
-  # when running with systemd, stop the systemd service first
-  $ sudo systemctl stop pgautofailover
-
-  # drop node ignores connection error to the monitor, and stops Postgres
-  $ pg_autoctl drop node
-
-  # register again, and restart Postgres on the node
-  $ pg_autoctl create postgres --monitor <new monitor uri> <--same options --as the --first time>
-
-  # when running with systemd, now start the systemd service again
-  $ sudo systemctl start pgautofailover
+  $ pg_autoctl disable monitor
+  $ pg_autoctl enable monitor

docs/operations.rst

@@ -51,7 +51,7 @@ As a result, here is the standard upgrade plan for pg_auto_failover:
   1. Upgrade the pg_auto_failover package on the all the nodes, monitor
      included.
 
-	 When using a debian based OS, this looks like the following command when 
+	 When using a debian based OS, this looks like the following command when
 	 from 1.4 to 1.5::
 
 	   sudo apt-get remove pg-auto-failover-cli-enterprise-1.4 postgresql-11-auto-failover-enterprise-1.4
@@ -361,6 +361,43 @@ The monitor reports every state change decision to a LISTEN/NOTIFY channel
 named ``state``. PostgreSQL logs on the monitor are also stored in a table,
 ``pgautofailover.event``, and broadcast by NOTIFY in the channel ``log``.
 
+.. _replacing_monitor_online:
+
+Replacing the monitor online
+----------------------------
+
+When the monitor node is not available anymore, it is possible to create a
+new monitor node and then switch existing nodes to a new monitor by using
+the following commands.
+
+  1. Apply the STONITH approach on the old monitor to make sure this node is
+     not going to show up again during the procedure. This step is sometimes
+     refered to as “fencing”.
+
+  2. On every node, ending with the (current) Postgres primary node for each
+     group, disable the monitor while ``pg_autoctl`` is still running::
+
+	   $ pg_autoctl disable monitor --force
+
+  3. Create a new monitor node::
+
+	   $ pg_autoctl create monitor ...
+
+  4. On the current primary node first, so that it's registered first and as
+     a primary still, for each group in your formation(s), enable the
+     monitor online again::
+
+	   $ pg_autoctl enable monitor --monitor postgresql://...
+
+  5. On every other (secondary) node, enable the monitor online again::
+
+	   $ pg_autoctl enable monitor --monitor postgresql://...
+
+This operation relies on the fact that a ``pg_autoctl`` can be operated
+without a monitor, and when reconnecting to a new monitor, this process
+reset the parts of the node state that comes from the monitor, such as the
+node identifier.
+
 Trouble-Shooting Guide
 ----------------------
 

docs/ref/reference.rst

@@ -76,11 +76,13 @@ keeper::
       secondary    Enable secondary nodes on a formation
       maintenance  Enable Postgres maintenance mode on this node
       ssl          Enable SSL configuration on this node
+      monitor      Enable a monitor for this node to be orchestrated from
 
     pg_autoctl disable
       secondary    Disable secondary nodes on a formation
       maintenance  Disable Postgres maintenance mode on this node
       ssl          Disable SSL configuration on this node
+      monitor      Disable the monitor for this node
 
     pg_autoctl get
     + node       get a node property from the pg_auto_failover monitor