Skip to content

Commit 057445b

Browse files
Eslam-NawaraashraffoudaNabil-Salah
authored
Update modules to update with config (#2583)
* update urls in internet module check * update bootstrap url in qemu makefile * fix hub url in debug image script * use new hub backend for g8ufs Signed-off-by: Ashraf Fouda <ashraf.m.fouda@gmail.com> * updates workflow to use the new hub Signed-off-by: Ashraf Fouda <ashraf.m.fouda@gmail.com> * return error if failed to initialize modules * load env when needed in registration * monitor changes in substrate url and update sub event listener accordingly * monitor changes in substrate and relay urls and update rmb peer accordingly * fix config update of api-gateway * check for updates only if substrate connection is broken * remove changes from deleted `fix-zos-urls` branch * allow relay update when changed * fix api-gateway update on substrate failure * update: hub_url binding to hub_urls Signed-off-by: nabil salah <nabil.salah203@gmail.com> * fix: hub_urls naming Signed-off-by: nabil salah <nabil.salah203@gmail.com> * update zosbase to latest * fixes relays connections of api_gateway Signed-off-by: Ashraf Fouda <ashraf.m.fouda@gmail.com> * fixes to substrate urls in noded Signed-off-by: Ashraf Fouda <ashraf.m.fouda@gmail.com> * removes getrelayurls() Signed-off-by: Ashraf Fouda <ashraf.m.fouda@gmail.com> * better linting Signed-off-by: Ashraf Fouda <ashraf.m.fouda@gmail.com> --------- Signed-off-by: Ashraf Fouda <ashraf.m.fouda@gmail.com> Signed-off-by: nabil salah <nabil.salah203@gmail.com> Co-authored-by: Ashraf Fouda <ashraf.m.fouda@gmail.com> Co-authored-by: nabil salah <nabil.salah203@gmail.com>
1 parent 90b88b6 commit 057445b

9 files changed

Lines changed: 134 additions & 17 deletions

File tree

.github/workflows/bin-package.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ jobs:
5656
./bins-extra.sh --package ${{ inputs.package }}
5757
- name: Publish flist (tf-autobuilder, ${{ steps.package.outputs.name }})
5858
if: success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/testdeploy' || startsWith(github.ref, 'refs/tags/v'))
59+
5960
uses: threefoldtech/publish-flist@master
6061
with:
6162
token: ${{ secrets.token }}

.github/workflows/publish.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ jobs:
7272
- name: Tagging
7373
uses: threefoldtech/publish-flist@master
7474
if: success() && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')|| github.ref == 'refs/heads/testdeploy')
75+
7576
with:
7677
token: ${{ secrets.HUB_JWT }}
7778
action: tag
@@ -83,6 +84,7 @@ jobs:
8384
# this basically releases this build to devnet
8485
- name: Cross tagging (development)
8586
if: success() && (github.ref == 'refs/heads/main'|| github.ref == 'refs/heads/testdeploy')
87+
8688
uses: threefoldtech/publish-flist@master
8789
with:
8890
token: ${{ secrets.HUB_JWT }}

bootstrap/bootstrap/src/hub.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@ use std::path::Path;
1111

1212
#[derive(Deserialize)]
1313
struct ZosConfig {
14-
hub_url: Vec<String>,
14+
hub_urls: Vec<String>,
1515
}
1616

17-
fn get_hub_url(runmode: &RunMode) -> Result<Vec<String>> {
17+
fn get_hub_urls(runmode: &RunMode) -> Result<Vec<String>> {
1818
let base_url = "https://github.com/threefoldtech/zos-config/raw/main/";
1919
let config_filename = match runmode {
2020
RunMode::Prod => "production.json",
@@ -43,10 +43,10 @@ fn get_hub_url(runmode: &RunMode) -> Result<Vec<String>> {
4343
Err(_) => return Ok(fallback),
4444
};
4545

46-
if config.hub_url.is_empty() {
46+
if config.hub_urls.is_empty() {
4747
Ok(fallback)
4848
} else {
49-
Ok(config.hub_url)
49+
Ok(config.hub_urls)
5050
}
5151
}
5252

@@ -86,7 +86,7 @@ impl Repo {
8686
T: AsRef<str>,
8787
{
8888
let config = crate::config::Config::current()?;
89-
let hub = get_hub_url(&config.runmode)?;
89+
let hub = get_hub_urls(&config.runmode)?;
9090
Ok(Repo {
9191
name: String::from(name.as_ref()),
9292
hub,

cmds/modules/api_gateway/main.go

Lines changed: 84 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"crypto/ed25519"
66
"encoding/hex"
77
"fmt"
8+
"slices"
9+
"time"
810

911
"github.com/cenkalti/backoff/v3"
1012
"github.com/rs/zerolog/log"
@@ -62,7 +64,9 @@ func action(cli *cli.Context) error {
6264
if err != nil {
6365
return err
6466
}
65-
67+
env := environment.MustGet()
68+
subURLs := env.SubstrateURL
69+
relayURLs := env.RelaysURLs
6670
manager, err := environment.GetSubstrate()
6771
if err != nil {
6872
return fmt.Errorf("failed to create substrate manager: %w", err)
@@ -105,17 +109,23 @@ func action(cli *cli.Context) error {
105109

106110
bo := backoff.NewExponentialBackOff()
107111
bo.MaxElapsedTime = 0
112+
113+
// this ctx is used to allow the node to restart the peer without leaving any unwanted open connections
114+
currentPeerCtx, cancel := context.WithCancel(ctx)
108115
backoff.Retry(func() error {
109116
_, err = peer.NewPeer(
110-
ctx,
117+
currentPeerCtx,
111118
hex.EncodeToString(pair.Seed()),
112119
manager,
113120
router.Serve,
114121
peer.WithKeyType(peer.KeyTypeEd25519),
115-
peer.WithRelay(environment.GetRelaysURLs()...),
122+
peer.WithRelay(relayURLs...),
116123
peer.WithInMemoryExpiration(6*60*60), // 6 hours
117124
)
118125
if err != nil {
126+
if cancel != nil {
127+
cancel()
128+
}
119129
return fmt.Errorf("failed to start a new rmb peer: %w", err)
120130
}
121131

@@ -128,6 +138,75 @@ func action(cli *cli.Context) error {
128138
Msg("starting api-gateway module")
129139

130140
// block forever
131-
<-ctx.Done()
132-
return nil
141+
for {
142+
select {
143+
case <-ctx.Done():
144+
return nil
145+
// check if we need to run an update on the peer and only do the update if all the changes are done successfully
146+
case <-time.After(10 * time.Minute):
147+
env, err := environment.Get()
148+
if err != nil {
149+
// skip update if we can't get env
150+
log.Debug().Err(err).Msg("failed to load node environment")
151+
continue
152+
}
153+
154+
updatedSubURLs := env.SubstrateURL
155+
updatedRelayURLs := env.RelaysURLs
156+
157+
// make sure urls are sorted for comparison
158+
slices.Sort(subURLs)
159+
slices.Sort(relayURLs)
160+
slices.Sort(updatedSubURLs)
161+
slices.Sort(updatedRelayURLs)
162+
163+
// skip update if substrate and relay urls did not change
164+
if slices.Equal(subURLs, updatedSubURLs) && slices.Equal(relayURLs, updatedRelayURLs) {
165+
log.Debug().Msg("zos-config doesn't have updated config to update the node with")
166+
continue
167+
}
168+
169+
log.Debug().Strs("relays_urls", updatedRelayURLs).Strs("substrate_urls", updatedSubURLs).Msg("detected new update in configuration")
170+
171+
manager, err = environment.GetSubstrate()
172+
if err != nil {
173+
// skip update if can't get sub manager
174+
log.Debug().Err(err).Msg("failed to get substrate manager")
175+
continue
176+
}
177+
// cancel the current peer to create new one with updated urls
178+
if cancel != nil {
179+
log.Debug().Msg("cancelling current peer context to create a new one with updated urls")
180+
cancel()
181+
}
182+
183+
currentPeerCtx, cancel = context.WithCancel(ctx)
184+
backoff.Retry(func() error {
185+
_, err = peer.NewPeer(
186+
currentPeerCtx,
187+
hex.EncodeToString(pair.Seed()),
188+
manager,
189+
router.Serve,
190+
peer.WithKeyType(peer.KeyTypeEd25519),
191+
peer.WithRelay(updatedRelayURLs...),
192+
peer.WithInMemoryExpiration(6*60*60), // 6 hours
193+
)
194+
if err != nil {
195+
if cancel != nil {
196+
197+
// cancel the context to avoid any unwanted open connections
198+
cancel()
199+
}
200+
return fmt.Errorf("failed to start a new rmb peer: %w", err)
201+
202+
}
203+
return nil
204+
}, bo)
205+
206+
relayURLs = updatedRelayURLs
207+
subURLs = updatedSubURLs
208+
209+
log.Debug().Strs("relays_urls", relayURLs).Strs("substrate_urls", subURLs).Msg("updated substrate and relay urls")
210+
}
211+
}
133212
}

cmds/modules/noded/main.go

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"crypto/ed25519"
66
"fmt"
7+
"slices"
78
"time"
89

910
"github.com/cenkalti/backoff"
@@ -58,7 +59,7 @@ var Module cli.Command = cli.Command{
5859
Action: action,
5960
}
6061

61-
func registerationServer(ctx context.Context, msgBrokerCon string, env environment.Environment, info registrar.RegistrationInfo) error {
62+
func registerationServer(ctx context.Context, msgBrokerCon string, info registrar.RegistrationInfo) error {
6263
redis, err := zbus.NewRedisClient(msgBrokerCon)
6364
if err != nil {
6465
return errors.Wrap(err, "fail to connect to message broker server")
@@ -69,7 +70,7 @@ func registerationServer(ctx context.Context, msgBrokerCon string, env environme
6970
return errors.Wrap(err, "fail to connect to message broker server")
7071
}
7172

72-
registrar := registrar.NewRegistrar(ctx, redis, env, info)
73+
registrar := registrar.NewRegistrar(ctx, redis, info)
7374
server.Register(zbus.ObjectID{Name: "registrar", Version: "0.0.1"}, registrar)
7475
log.Debug().Msg("object registered")
7576
if err := server.Run(ctx); err != nil && err != context.Canceled {
@@ -85,6 +86,7 @@ func action(cli *cli.Context) error {
8586
printNet bool = cli.Bool("net")
8687
)
8788
env := environment.MustGet()
89+
subURLs := env.SubstrateURL
8890

8991
redis, err := zbus.NewRedisClient(msgBrokerCon)
9092
if err != nil {
@@ -159,7 +161,7 @@ func action(cli *cli.Context) error {
159161
WithSecureBoot(secureBoot).
160162
WithVirtualized(len(hypervisor) != 0)
161163

162-
go registerationServer(ctx, msgBrokerCon, env, info)
164+
go registerationServer(ctx, msgBrokerCon, info)
163165
log.Info().Msg("start perf scheduler")
164166

165167
perfMon, err := perf.NewPerformanceMonitor(msgBrokerCon)
@@ -245,6 +247,38 @@ func action(cli *cli.Context) error {
245247
}
246248
}()
247249

250+
// monitor node env updates in substrate url
251+
go func() {
252+
for {
253+
<-time.After(10 * time.Minute)
254+
255+
env, err = environment.Get()
256+
if err != nil {
257+
log.Debug().Err(err).Msg("failed to get updated config")
258+
continue
259+
}
260+
261+
newSubURLs := env.SubstrateURL
262+
slices.Sort(subURLs)
263+
slices.Sort(newSubURLs)
264+
265+
if slices.Equal(subURLs, newSubURLs) {
266+
log.Debug().Msg("zos-config doesn't have updated config to update the node with")
267+
continue
268+
}
269+
log.Info().Msg("zos-config has updated substrate urls, updating the node with the new urls in noded")
270+
sub, err := environment.GetSubstrate()
271+
if err != nil {
272+
log.Debug().Err(err).Msg("failed to get updated substrate manager")
273+
continue
274+
}
275+
276+
subURLs = env.SubstrateURL
277+
events.UpdateSubstrateManager(sub)
278+
log.Debug().Strs("substrate_urls", env.SubstrateURL).Msg("updated substrate events handler to use new substrate urls")
279+
}
280+
}()
281+
248282
log.Info().Uint32("twin", twin).Msg("node has been registered")
249283
idStub := stubs.NewIdentityManagerStub(redis)
250284
fetchCtx, cancel := context.WithTimeout(ctx, 30*time.Second)

cmds/modules/provisiond/main.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,7 @@ func action(cli *cli.Context) error {
365365
// call the runtime upgrade before running engine
366366
if err := provisioners.Initialize(ctx); err != nil {
367367
log.Error().Err(err).Msg("failed to run provisioners initializers")
368+
return err
368369
}
369370

370371
// spawn the engine

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ require (
1717
github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20250703093252-e7500b106618
1818
github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.16.9-0.20250820184355-b0364f34a442
1919
github.com/threefoldtech/zbus v1.0.1
20-
github.com/threefoldtech/zosbase v0.1.11-0.20250729092219-8aa23b682790
20+
github.com/threefoldtech/zosbase v0.1.11-0.20250819141009-370d63ff91fc
2121
github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa
2222
gopkg.in/yaml.v2 v2.4.0
2323
)

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -562,8 +562,8 @@ github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.16.9-0.20250820184355-b0364
562562
github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.16.9-0.20250820184355-b0364f34a442/go.mod h1:D3m+WMX2eR5B5ZotOwHshYAxfzmA5QTfFH7kabe51a8=
563563
github.com/threefoldtech/zbus v1.0.1 h1:3KaEpyOiDYAw+lrAyoQUGIvY9BcjVRXlQ1beBRqhRNk=
564564
github.com/threefoldtech/zbus v1.0.1/go.mod h1:E/v/xEvG/l6z/Oj0aDkuSUXFm/1RVJkhKBwDTAIdsHo=
565-
github.com/threefoldtech/zosbase v0.1.11-0.20250729092219-8aa23b682790 h1:8b3sjTH1QQn1ciOm458idHEjgPXacYnU9C3xgX8LtTw=
566-
github.com/threefoldtech/zosbase v0.1.11-0.20250729092219-8aa23b682790/go.mod h1:v3moVR/JyNGG8LhiPhq6oPyAYdVL+AluCsmj4c0EzfA=
565+
github.com/threefoldtech/zosbase v0.1.11-0.20250819141009-370d63ff91fc h1:Pck+H7JU+a7JTS2jm2Qcv478+ZArN17Mgb6+czu/tnk=
566+
github.com/threefoldtech/zosbase v0.1.11-0.20250819141009-370d63ff91fc/go.mod h1:MzVVpnslwa4UOzE0YaAbl99ZYLqo5hC+KVjOsVquPlA=
567567
github.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0=
568568
github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
569569
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=

qemu/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,4 @@ run:
3030

3131
run-gpu:
3232
@echo "Running your node"
33-
sudo ./vm_gpu.sh -g -n node-01 -c "farmer_id=$(id) version=v3 printk.devmsg=on runmode=dev nomodeset ssh-user=$(user)"
33+
sudo ./vm_gpu.sh -g -n node-01 -c "farmer_id=$(id) version=v3 printk.devmsg=on runmode=dev nomodeset ssh-user=$(user)"

0 commit comments

Comments
 (0)