merged master, resolved conflict

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

Author: shlomi-noach

.github/workflows/ci.yml

@@ -10,10 +10,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Go 1.16
+    - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.16
+        go-version: 1.17
 
     - name: Build
       run: script/cibuild

.github/workflows/codeql.yml

@@ -2,6 +2,7 @@ name: "CodeQL analysis"
 
 on: 
   push:
+  pull_request:
   schedule:
     - cron: '0 0 * * 0'
 

.github/workflows/golangci-lint.yml

@@ -0,0 +1,23 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.17
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.46.2

.github/workflows/replica-tests.yml

@@ -8,15 +8,15 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        version: [mysql-5.5.62,mysql-5.6.43,mysql-5.7.25,mysql-8.0.16]
+        version: [mysql-5.7.25,mysql-8.0.16]
 
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Go 1.16
+    - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.16
+        go-version: 1.17
 
     - name: migration tests
       env:

.golangci.yml

@@ -0,0 +1,24 @@
+run:
+  timeout: 5m
+linters:
+  disable:
+    - errcheck
+  enable:
+    - contextcheck
+    - durationcheck
+    - errname
+    - execinquery
+    - gofmt
+    - ifshort
+    - misspell
+    - nilerr
+    - noctx
+    - nolintlint
+    - nosprintfhostport
+    - prealloc
+    - rowserrcheck
+    - sqlclosecheck
+    - unconvert
+    - unused
+    - wastedassign
+    - whitespace

Dockerfile.packaging

@@ -1,4 +1,4 @@
-FROM golang:1.16.4
+FROM golang:1.17
 
 RUN apt-get update
 RUN apt-get install -y ruby ruby-dev rubygems build-essential

Dockerfile.test

@@ -1,4 +1,4 @@
-FROM golang:1.16.4
+FROM golang:1.17
 LABEL maintainer="github@github.com"
 
 RUN apt-get update

build.sh

@@ -18,30 +18,32 @@ function build {
   GOOS=$3
   GOARCH=$4
 
-  if ! go version | egrep -q 'go(1\.1[567])' ; then
+  if ! go version | egrep -q 'go1\.(1[5-9]|[2-9][0-9]{1})' ; then
     echo "go version must be 1.15 or above"
     exit 1
   fi
 
-  echo "Building ${osname} binary"
+  echo "Building ${osname}-${GOARCH} binary"
   export GOOS
   export GOARCH
   go build -ldflags "$ldflags" -o $buildpath/$target go/cmd/gh-ost/main.go
 
   if [ $? -ne 0 ]; then
-      echo "Build failed for ${osname}"
+      echo "Build failed for ${osname} ${GOARCH}."
       exit 1
   fi
 
-  (cd $buildpath && tar cfz ./gh-ost-binary-${osshort}-${timestamp}.tar.gz $target)
+  (cd $buildpath && tar cfz ./gh-ost-binary-${osshort}-${GOARCH}-${timestamp}.tar.gz $target)
 
-  if [ "$GOOS" == "linux" ] ; then
+  # build RPM and deb for Linux, x86-64 only
+  if [ "$GOOS" == "linux" ] && [ "$GOARCH" == "amd64" ] ; then
     echo "Creating Distro full packages"
     builddir=$(setuptree)
     cp $buildpath/$target $builddir/gh-ost/usr/bin
     cd $buildpath
-    fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach <shlomi-noach+gh-ost-deb@github.com>' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" .
-    fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach <shlomi-noach+gh-ost-deb@github.com>' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files .
+    fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'GitHub' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" --rpm-os linux .
+    fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'GitHub' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files .
+    cd -
   fi
 }
 
@@ -62,10 +64,15 @@ main() {
   mkdir -p ${buildpath}
   rm -rf ${buildpath:?}/*
   build GNU/Linux linux linux amd64
-  # build macOS osx darwin amd64
+  build GNU/Linux linux linux arm64
+  build macOS osx darwin amd64
+  build macOS osx darwin arm64
 
   echo "Binaries found in:"
   find $buildpath/gh-ost* -type f -maxdepth 1
+
+  echo "Checksums:"
+  (cd $buildpath && shasum -a256 gh-ost* 2>/dev/null)
 }
 
 main "$@"

doc/coding-ghost.md

@@ -5,7 +5,7 @@
 Getting started with gh-ost development is simple!
 
 - First obtain the repository with `git clone` or `go get`.
-- From inside of the repository run `script/cibuild`
+- From inside of the repository run `script/cibuild`.
 - This will bootstrap the environment if needed, format the code, build the code, and then run the unit test.
 
 ## CI build workflow
@@ -14,6 +14,12 @@ Getting started with gh-ost development is simple!
 
 If additional steps are needed, please add them into this workflow so that the workflow remains simple.
 
+## `golang-ci` linter
+
+To enfore best-practices, Pull Requests are automatically linted by [`golang-ci`](https://golangci-lint.run/). The linter config is located at [`.golangci.yml`](https://github.com/github/gh-ost/blob/master/.golangci.yml) and the `golangci-lint` GitHub Action is located at [`.github/workflows/golangci-lint.yml`](https://github.com/github/gh-ost/blob/master/.github/workflows/golangci-lint.yml).
+
+To run the `golang-ci` linters locally _(recommended before push)_, use `script/lint`.
+
 ## Notes:
 
 Currently, `script/ensure-go-installed` will install `go` for Mac OS X and Linux. We welcome PR's to add other platforms.

doc/command-line-flags.md

@@ -26,7 +26,7 @@ If, for some reason, you do not wish `gh-ost` to connect to a replica, you may c
 
 ### approve-renamed-columns
 
-When your migration issues a column rename (`change column old_name new_name ...`) `gh-ost` analyzes the statement to try and associate the old column name with new column name. Otherwise the new structure may also look like some column was dropped and another was added.
+When your migration issues a column rename (`change column old_name new_name ...`) `gh-ost` analyzes the statement to try and associate the old column name with new column name. Otherwise, the new structure may also look like some column was dropped and another was added.
 
 `gh-ost` will print out what it thinks the _rename_ implied, but will not issue the migration unless you provide with `--approve-renamed-columns`.
 
@@ -36,7 +36,7 @@ If you think `gh-ost` is mistaken and that there's actually no _rename_ involved
 
 `gh-ost` infers the identity of the master server by crawling up the replication topology. You may explicitly tell `gh-ost` the identity of the master host via `--assume-master-host=the.master.com`. This is useful in:
 
-- _master-master_ topologies (together with [`--allow-master-master`](#allow-master-master)), where `gh-ost` can arbitrarily pick one of the co-masters and you prefer that it picks a specific one
+- _master-master_ topologies (together with [`--allow-master-master`](#allow-master-master)), where `gh-ost` can arbitrarily pick one of the co-masters, and you prefer that it picks a specific one
 - _tungsten replicator_ topologies (together with [`--tungsten`](#tungsten)), where `gh-ost` is unable to crawl and detect the master
 
 ### assume-rbr
@@ -65,7 +65,13 @@ Comma delimited status-name=threshold, same format as [`--max-load`](#max-load).
 
 `--critical-load` defines a threshold that, when met, `gh-ost` panics and bails out. The default behavior is to bail out immediately when meeting this threshold.
 
-This may sometimes lead to migrations bailing out on a very short spike, that, while in itself is impacting production and is worth investigating, isn't reason enough to kill a 10 hour migration.
+This may sometimes lead to migrations bailing out on a very short spike, that, while in itself is impacting production and is worth investigating, isn't reason enough to kill a 10-hour migration.
+
+### critical-load-hibernate-seconds
+
+When `--critical-load-hibernate-seconds` is non-zero (e.g. `--critical-load-hibernate-seconds=300`), `critical-load` does not panic and bail out; instead, `gh-ost` goes into hibernation for the specified duration. It will not read/write anything from/to any server during this time.  Execution then continues upon waking from hibernation.
+
+If `critical-load` is met again, `gh-ost` will repeat this cycle, and never panic and bail out.
 
 ### critical-load-interval-millis
 
@@ -102,7 +108,7 @@ Noteworthy is that setting `--dml-batch-size` to higher value _does not_ mean `g
 
 ### exact-rowcount
 
-A `gh-ost` execution need to copy whatever rows you have in your existing table onto the ghost table. This can, and often be, a large number. Exactly what that number is?
+A `gh-ost` execution need to copy whatever rows you have in your existing table onto the ghost table. This can and often will be, a large number. Exactly what that number is?
 `gh-ost` initially estimates the number of rows in your table by issuing an `explain select * from your_table`. This will use statistics on your table and return with a rough estimate. How rough? It might go as low as half or as high as double the actual number of rows in your table. This is the same method as used in [`pt-online-schema-change`](https://www.percona.com/doc/percona-toolkit/2.2/pt-online-schema-change.html).
 
 `gh-ost` also supports the `--exact-rowcount` flag. When this flag is given, two things happen:
@@ -139,6 +145,10 @@ Add this flag when executing on a 1st generation Google Cloud Platform (GCP).
 
 Default 100. See [`subsecond-lag`](subsecond-lag.md) for details.
 
+### hooks-status-interval
+
+Defaults to 60 seconds. Configures how often the `gh-ost-on-status` hook is called, see [`hooks`](hooks.md) for full details on how to use hooks.
+
 ### initially-drop-ghost-table
 
 `gh-ost` maintains two tables while migrating: the _ghost_ table (which is synced from your original table and finally replaces it) and a changelog table, which is used internally for bookkeeping. By default, it panics and aborts if it sees those tables upon startup. Provide `--initially-drop-ghost-table` and `--initially-drop-old-table` to let `gh-ost` know it's OK to drop them beforehand.
@@ -234,7 +244,15 @@ Provide a command delimited list of replicas; `gh-ost` will throttle when any of
 
 ### throttle-http
 
-Provide a HTTP endpoint; `gh-ost` will issue `HEAD` requests on given URL and throttle whenever response status code is not `200`. The URL can be queried and updated dynamically via [interactive commands](interactive-commands.md). Empty URL disables the HTTP check.
+Provide an HTTP endpoint; `gh-ost` will issue `HEAD` requests on given URL and throttle whenever response status code is not `200`. The URL can be queried and updated dynamically via [interactive commands](interactive-commands.md). Empty URL disables the HTTP check.
+
+### throttle-http-interval-millis
+
+Defaults to 100. Configures the HTTP throttle check interval in milliseconds.
+
+### throttle-http-timeout-millis
+
+Defaults to 1000 (1 second). Configures the HTTP throttler check timeout in milliseconds.
 
 ### timestamp-old-table