fix api-gateway update on substrate failure

Author: Eslam-Nawara

cmds/modules/api_gateway/main.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v3"
-	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
 	"github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go/peer"
@@ -144,7 +143,16 @@ func action(cli *cli.Context) error {
 			return nil
 			// check if we need to run an update on the peer and only do the update if all the changes are done successfully
 		case <-time.After(10 * time.Minute):
-			log.Debug().Msg("checking for updates in zos config")
+
+			log.Debug().Msg("checking if node is maintaining a healty substrate connection")
+			cl, _, err := manager.Raw()
+			if err == nil {
+				// skip update if the connection is working properly
+				log.Debug().Msg("the open connection is healty, no update needed")
+				cl.Client.Close()
+				continue
+			}
+
 			env, err := environment.Get()
 			if err != nil {
 				// skip update if we can't get env
@@ -163,26 +171,35 @@ func action(cli *cli.Context) error {
 
 			// skip update if substrate and relay urls did not change
 			if slices.Equal(subURLs, updatedSubURLs) && slices.Equal(relayURLs, updatedRelayURLs) {
+				log.Debug().Msg("zos-config doesn't have updated config to update the node with")
 				continue
 			}
 
-			// skip update if the old substrate connection is working fine
-			if !slices.Equal(subURLs, updatedSubURLs) {
-				if _, err := gw.GetTime(); err == nil {
-					log.Debug().Msg("the open connection is already working, no update needed")
-					continue
-				}
+			log.Debug().Strs("relays_urls", updatedRelayURLs).Strs("substrate_urls", updatedSubURLs).Msg("detected new update in configuration")
+			newPeerCtx, newCancel := context.WithCancel(ctx)
+
+			newManager, err := environment.GetSubstrate()
+			if err != nil {
+				// skip update if can't get sub manager
+				log.Debug().Err(err).Msg("failed to get substrate manager")
+				continue
 			}
 
-			newPeerCtx, newCancel := context.WithCancel(ctx)
-			if err := updatePeer(newPeerCtx, updatedSubURLs, updatedRelayURLs, pair.Seed(), router.Serve); err != nil {
-				log.Debug().Err(err).Send()
-				newCancel()
+			if _, err = peer.NewPeer(
+				newPeerCtx,
+				hex.EncodeToString(pair.Seed()),
+				newManager,
+				router.Serve,
+				peer.WithKeyType(peer.KeyTypeEd25519),
+				peer.WithRelay(updatedRelayURLs...),
+				peer.WithInMemoryExpiration(6*60*60), // 6 hours
+			); err != nil {
+				log.Debug().Err(err).Msg("failed to start a new rmb peer")
 				continue
 			}
 
 			if !slices.Equal(subURLs, updatedSubURLs) {
-				err = gw.UpdateSubstrateGatewayConnection(manager)
+				err = gw.UpdateSubstrateGatewayConnection(newManager)
 				if err != nil {
 					log.Debug().Err(err).Msg("failed to update substrate gateway with new manager")
 					newCancel()
@@ -194,33 +211,11 @@ func action(cli *cli.Context) error {
 			peerCtx = newPeerCtx
 			cancel = newCancel
 
+			manager = newManager
 			relayURLs = updatedRelayURLs
 			subURLs = updatedSubURLs
+
 			log.Debug().Strs("relays_urls", relayURLs).Strs("substrate_urls", subURLs).Msg("updated substrate and relay urls")
 		}
 	}
 }
-
-func updatePeer(ctx context.Context, subURLs, relayURLs []string, seed []byte, serve peer.Handler) error {
-	log.Debug().Strs("relays_urls", relayURLs).Strs("substrate_urls", subURLs).Msg("detected new update in configuration")
-
-	manager, err := environment.GetSubstrate()
-	if err != nil {
-		// skip update if can't get sub manager
-		return errors.Wrap(err, "failed to get substrate manager")
-	}
-
-	if _, err = peer.NewPeer(
-		ctx,
-		hex.EncodeToString(seed),
-		manager,
-		serve,
-		peer.WithKeyType(peer.KeyTypeEd25519),
-		peer.WithRelay(relayURLs...),
-		peer.WithInMemoryExpiration(6*60*60), // 6 hours
-	); err != nil {
-		return errors.Wrap(err, "failed to start a new rmb peer")
-	}
-
-	return nil
-}

cmds/modules/noded/main.go

@@ -252,14 +252,15 @@ func action(cli *cli.Context) error {
 		for {
 			<-time.After(10 * time.Minute)
 
-			log.Debug().Msg("checking for updates in zos config")
+			log.Debug().Msg("checking if node is maintaining a healty substrate connection")
 			cl, _, err := sub.Raw()
 			if err == nil {
 				// skip update if the connection is working properly
-				log.Debug().Msg("the open connection is working, no update needed")
+				log.Debug().Msg("the open connection is healty, no update needed")
 				cl.Client.Close()
 				continue
 			}
+			log.Debug().Err(err).Msg("used substrate manager is not healty, trying to update it")
 
 			// update only if the substrate connection is not healty
 			newEnv, err := environment.Get()
@@ -272,19 +273,21 @@ func action(cli *cli.Context) error {
 			slices.Sort(subURLs)
 			slices.Sort(newSubURLs)
 
-			if !slices.Equal(subURLs, newSubURLs) {
-				newSub, err := environment.GetSubstrate()
-				if err != nil {
-					log.Debug().Err(err).Msg("failed to get updated substrate manager")
-					continue
-				}
-
-				sub = newSub
-				subURLs = newSubURLs
-				events.UpdateSubstrateManager(sub)
-				log.Debug().Strs("substrate_urls", newEnv.SubstrateURL).Msg("updated substrate events handler to use new substrate urls")
+			if slices.Equal(subURLs, newSubURLs) {
+				log.Debug().Msg("zos-config doesn't have updated config to update the node with")
+				continue
+			}
 
+			newSub, err := environment.GetSubstrate()
+			if err != nil {
+				log.Debug().Err(err).Msg("failed to get updated substrate manager")
+				continue
 			}
+
+			sub = newSub
+			subURLs = newSubURLs
+			events.UpdateSubstrateManager(sub)
+			log.Debug().Strs("substrate_urls", newEnv.SubstrateURL).Msg("updated substrate events handler to use new substrate urls")
 		}
 	}()