Use our docker test infrastructure on GitHub Actions. (#937)

* Use our docker test infrastructure on GitHub Actions.

This simplifies the maintenance of our test infra, allowing to run the same
tests on the local development environment and in the automated CI.

Because citus_indent depends on a local git checkout to be able to run
correctly, we still need a special target for this test, and we still need
to manually install citusdata/tools in the test VM directly.

* Tests run in docker do not need an interactive environment.

* Fix running "linting" and "tablespaces" tests.

* Install missing linting tools (Python black).

* Improve ci testing integration, use more of the Makefile.

Author: DimCitus

.github/workflows/build-package.yml

@@ -34,10 +34,8 @@ jobs:
         include:
           - PGVERSION: 14
             TEST: tablespaces
-            DOCKERTEST: true
           - PGVERSION: 14
             TEST: linting
-            LINTING: true
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
@@ -46,44 +44,21 @@ jobs:
         run: |
             echo "PGVERSION=${{ matrix.PGVERSION }}" >> $GITHUB_ENV
             echo "TEST=${{ matrix.TEST }}" >> $GITHUB_ENV
-            echo "DOCKERTEST=${{ matrix.DOCKERTEST }}" >> $GITHUB_ENV
             echo "LINTING=${{ matrix.LINTING }}" >> $GITHUB_ENV
             echo "TRAVIS_BUILD_DIR=$(pwd)" >> $GITHUB_ENV
 
-      - name: Clone and install tools branch
+      - name: Clone and install linting tools
         run: |
+          sudo apt-get install python3-pip
+          pip3 install --user black
+          black --version
           gcc --version
           git clone -b v0.8.19 --depth 1 https://github.com/citusdata/tools.git ../tools
           sudo make -C ../tools install
+          install_uncrustify
+          rm -rf uncrustify*
 
-      - name: Pre Install steps
-        run: |
-          if [ -z "${LINTING}" ]; then setup_apt; fi
-          if [ -z "${LINTING}" ]; then nuke_pg; fi
-          python --version
-          python3 --version
-          sudo apt-get install liblz4-1 liblz4-dev bridge-utils python3-pip python3-nose python3-psycopg2 libxslt1-dev libzstd-dev
-          sudo apt-get install libkrb5-dev python3-setuptools
-          sudo -H pip3 install pyroute2>=0.5.17
-          pip3 install --user black
-
-      - name: Install steps
-        run: |
-         if [ -n "${LINTING}" ]; then install_uncrustify; fi
-         if [ -n "${LINTING}" ]; then rm -rf uncrustify*; fi
-         if [ -z "${LINTING}" ]; then install_pg; fi
-         if [ -z "${LINTING}" ]; then install_custom_pg; fi
-         env
-         if [ -z "${LINTING}" ]; then pg_config; fi
-         if [ -z "${LINTING}" ]; then PATH=`pg_config --bindir`:$PATH which pg_ctl; fi
-
-      - name: Script steps
+      - name: Run Test
         timeout-minutes: 15
         run: |
-          echo "Travis build dir: ${TRAVIS_BUILD_DIR}"
-          if [ -n "${LINTING}" ]; then citus_indent --check; fi
-          if [ -n "${LINTING}" ]; then black --check .; fi
-          if [ -n "${LINTING}" ]; then ci/banned.h.sh; fi
-          if [ -z "${LINTING}" ]; then make -j5 CFLAGS=-Werror; fi
-          if [ -z "${LINTING}" ]; then sudo make install; fi
-          if [ -z "${LINTING}" ]; then PATH=`pg_config --bindir`:$PATH make test; fi
+          make ci-test

Makefile

@@ -6,7 +6,7 @@ TOP := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 CONTAINER_NAME = pg_auto_failover
 BUILD_CONTAINER_NAME = pg_auto_failover_build
 TEST_CONTAINER_NAME = pg_auto_failover_test
-DOCKER_RUN_OPTS = --privileged  -ti --rm
+DOCKER_RUN_OPTS = --privileged --rm
 
 PGVERSION ?= 10
 NOSETESTS = $(shell which nosetests3 || which nosetests)
@@ -132,10 +132,31 @@ clean-bin:
 install-bin: bin
 	$(MAKE) -C src/bin/ install
 
+#
+# make ci-test; is run on the GitHub Action workflow
+#
+# In that environment we have a local git checkout of the code, and docker
+# is available too. We run our tests in docker, except for the code linting
+# parts which requires full access to the git repository, so linter tooling
+# is installed directly on the CI vm.
+#
+ci-test:
+ifeq ($(TEST),tablespaces)
+	$(MAKE) -C tests/tablespaces run-test
+else ifeq ($(TEST),linting)
+	$(MAKE) spellcheck
+else
+	$(MAKE) run-test
+endif
 
+#
+# make test; is run from inside the testing Docker image.
+#
 test:
 ifeq ($(TEST),tablespaces)
 	$(MAKE) -C tests/tablespaces run-test
+else ifeq ($(TEST),linting)
+	$(MAKE) spellcheck
 else
 	sudo -E env "PATH=${PATH}" USER=$(shell whoami) \
 		$(NOSETESTS)			\
@@ -146,10 +167,29 @@ else
 		${TEST_ARGUMENT}
 endif
 
+#
+# make indent; edits the code when necessary
+#
 indent:
 	citus_indent
 	black .
 
+#
+# make lint; is an alias for make spellcheck
+# make linting; is an alias for make spellcheck
+#
+lint: spellcheck ;
+linting: spellcheck ;
+
+#
+# make spellcheck; runs our linting tools without editing the code, only
+# reports compliance with the rules.
+#
+spellcheck:
+	citus_indent --check
+	black --check .
+	ci/banned.h.sh
+
 docs: $(FSM)
 	$(MAKE) -C docs html
 
@@ -177,6 +217,10 @@ build-test-pg14:
 build-test-pg15:
 	docker build --build-arg PGVERSION=15 --target test -t $(TEST_CONTAINER_NAME):pg15 .
 
+#
+# make run-test; is the main testing entry point used to run tests inside
+# our testing Docker container. The docker container depends on PGVERSION.
+#
 run-test: build-test-pg$(PGVERSION)
 	docker run					                \
 		--name $(TEST_CONTAINER_NAME)		    \
@@ -330,7 +374,7 @@ azdrop: all
 .PHONY: all clean check install docs
 .PHONY: monitor clean-monitor check-monitor install-monitor
 .PHONY: bin clean-bin install-bin
-.PHONY: build-test run-test
+.PHONY: build-test run-test spellcheck lint linting
 .PHONY: tmux-clean cluster compose
 .PHONY: azcluster azdrop az
 .PHONY: build-image