add backoff retry on service check (#2262)

* add backoff retry on service check to pass time if the network is not setup yet

* - directly push errors to zui after running each check
- move the retrying to an upper level to retry the whole check with pushing to zui

* cleaner err check

* run each health check in separate routine

Author: Omarabdul3ziz

pkg/perf/healthcheck/healthcheck.go

@@ -4,7 +4,10 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"sync"
+	"time"
 
+	"github.com/cenkalti/backoff"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/perf"
@@ -60,24 +63,49 @@ func (h *healthcheckTask) Run(ctx context.Context) (interface{}, error) {
 	log.Debug().Msg("starting health check task")
 	errs := make(map[string][]string)
 
+	cl := perf.GetZbusClient(ctx)
+	zui := stubs.NewZUIStub(cl)
+
+	var wg sync.WaitGroup
+	var mut sync.Mutex
 	for label, check := range h.checks {
-		errors := check(ctx)
-		if len(errors) == 0 {
-			continue
-		}
+		wg.Add(1)
 
-		errs[label] = errorsToStrings(errors)
-	}
+		go func(label string, check checkFunc) {
+			defer wg.Done()
 
-	cl := perf.GetZbusClient(ctx)
-	zui := stubs.NewZUIStub(cl)
+			op := func() error {
+				errors := check(ctx)
 
-	for label := range h.checks {
-		err := zui.PushErrors(ctx, label, errs[label])
-		if err != nil {
-			return nil, err
-		}
+				mut.Lock()
+				defer mut.Unlock()
+				errs[label] = errorsToStrings(errors)
+
+				if err := zui.PushErrors(ctx, label, errs[label]); err != nil {
+					return err
+				}
+
+				if len(errors) != 0 {
+					return fmt.Errorf("failed health check")
+				}
+
+				return nil
+			}
+
+			notify := func(err error, t time.Duration) {
+				log.Error().Err(err).Str("check", label).Dur("retry-in", t).Msg("failed health check. retrying")
+			}
+
+			bo := backoff.NewExponentialBackOff()
+			bo.InitialInterval = 30 * time.Second
+			bo.MaxInterval = 30 * time.Second
+			bo.MaxElapsedTime = 10 * time.Minute
+
+			_ = backoff.RetryNotify(op, bo, notify)
+		}(label, check)
 	}
+	wg.Wait()
+
 	return errs, nil
 }