Skip to content

Commit 04ba7a3

Browse files
Nabil-SalahNabil-Salah
authored
Switch to retriable HTTP (#2515)
* feat: use of retryable http server Signed-off-by: nabil salah <nabil.salah203@gmail.com> * feat: use retryable http in zos upgarde hub and increase the max retry count Signed-off-by: nabil salah <nabil.salah203@gmail.com> * remove: request wrapping Signed-off-by: nabil salah <nabil.salah203@gmail.com> * feat: use retryable http on more clients Signed-off-by: nabil salah <nabil.salah203@gmail.com> * fix:linting --------- Signed-off-by: nabil salah <nabil.salah203@gmail.com>
1 parent e6667ae commit 04ba7a3

12 files changed

Lines changed: 81 additions & 58 deletions

File tree

‎client/_http/node.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"net/http"
1010
"path/filepath"
1111

12+
"github.com/hashicorp/go-retryablehttp"
1213
"github.com/pkg/errors"
1314
"github.com/threefoldtech/zos/pkg/gridtypes"
1415
)
@@ -85,7 +86,7 @@ func (n *NodeClient) Deploy(dl *gridtypes.Deployment, update bool) error {
8586
return errors.Wrap(err, "failed to sign request")
8687
}
8788

88-
response, err := http.DefaultClient.Do(request)
89+
response, err := retryablehttp.DefaultClient.Do(request)
8990
if err != nil {
9091
return err
9192
}
@@ -110,7 +111,7 @@ func (n *NodeClient) Get(twin, deployment uint32) (dl gridtypes.Deployment, err
110111
return dl, errors.Wrap(err, "failed to sign request")
111112
}
112113

113-
response, err := http.DefaultClient.Do(request)
114+
response, err := retryablehttp.DefaultClient.Do(request)
114115
if err != nil {
115116
return dl, err
116117
}
@@ -135,7 +136,7 @@ func (n *NodeClient) Delete(twin, deployment uint32) (err error) {
135136
return errors.Wrap(err, "failed to sign request")
136137
}
137138

138-
response, err := http.DefaultClient.Do(request)
139+
response, err := retryablehttp.DefaultClient.Do(request)
139140
if err != nil {
140141
return err
141142
}
@@ -155,7 +156,7 @@ func (n *NodeClient) Counters() (total gridtypes.Capacity, used gridtypes.Capaci
155156
return total, used, errors.Wrap(err, "failed to build request")
156157
}
157158

158-
response, err := http.DefaultClient.Do(request)
159+
response, err := retryablehttp.DefaultClient.Do(request)
159160
if err != nil {
160161
return total, used, err
161162
}

‎cmds/identityd/ssh.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"time"
1111

1212
"github.com/cenkalti/backoff"
13+
"github.com/hashicorp/go-retryablehttp"
1314
"github.com/rs/zerolog/log"
1415
"github.com/threefoldtech/zos/pkg"
1516
"github.com/threefoldtech/zos/pkg/environment"
@@ -72,7 +73,7 @@ func manageSSHKeys() error {
7273

7374
for _, user := range authorizedUsers {
7475
fetchKey := func() error {
75-
res, err := http.Get(fmt.Sprintf("https://github.com/%s.keys", user))
76+
res, err := retryablehttp.Get(fmt.Sprintf("https://github.com/%s.keys", user))
7677

7778
if err != nil {
7879
return fmt.Errorf("failed to fetch user keys: %+w", err)

‎go.mod‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ require (
6262
gotest.tools v2.2.0+incompatible
6363
)
6464

65+
require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
66+
6567
require (
6668
github.com/Microsoft/go-winio v0.5.2 // indirect
6769
github.com/Microsoft/hcsshim v0.8.25 // indirect
@@ -94,6 +96,7 @@ require (
9496
github.com/hanwen/go-fuse/v2 v2.3.0 // indirect
9597
github.com/hashicorp/errwrap v1.1.0 // indirect
9698
github.com/hashicorp/go-multierror v1.1.1 // indirect
99+
github.com/hashicorp/go-retryablehttp v0.7.7
97100
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
98101
github.com/holiman/uint256 v1.2.3 // indirect
99102
github.com/jarcoal/httpmock v1.3.1

‎go.sum‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,8 +287,12 @@ github.com/hanwen/go-fuse/v2 v2.3.0/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1r
287287
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
288288
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
289289
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
290+
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
291+
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
290292
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
291293
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
294+
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
295+
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
292296
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
293297
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
294298
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=

‎pkg/environment/config.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"strings"
1010
"time"
1111

12+
"github.com/hashicorp/go-retryablehttp"
1213
"github.com/pkg/errors"
1314
)
1415

@@ -51,11 +52,11 @@ func GetConfig() (base Config, err error) {
5152

5253
// GetConfig returns extend config for specific run mode
5354
func GetConfigForMode(mode RunMode) (Config, error) {
54-
httpClient := &http.Client{
55-
Timeout: defaultHttpTimeout,
56-
}
55+
httpClient := retryablehttp.NewClient()
56+
httpClient.HTTPClient.Timeout = defaultHttpTimeout
57+
httpClient.RetryMax = 5
5758

58-
return getConfig(mode, baseExtendedURL, httpClient)
59+
return getConfig(mode, baseExtendedURL, httpClient.StandardClient())
5960
}
6061

6162
func uniqueStr(slice []string) []string {

‎pkg/flist/flist.go‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"time"
1919

2020
"github.com/containernetworking/plugins/pkg/ns"
21+
"github.com/hashicorp/go-retryablehttp"
2122
"github.com/pkg/errors"
2223
"github.com/rs/zerolog/log"
2324
"github.com/threefoldtech/zos/pkg"
@@ -126,7 +127,7 @@ type flistModule struct {
126127
commander commander
127128
system system
128129

129-
httpClient *http.Client
130+
httpClient *retryablehttp.Client
130131
}
131132

132133
func newFlister(root string, storage volumeAllocator, commander commander, system system) *flistModule {
@@ -155,6 +156,9 @@ func newFlister(root string, storage volumeAllocator, commander commander, syste
155156
}
156157
}
157158

159+
httpClient := retryablehttp.NewClient()
160+
httpClient.HTTPClient.Timeout = defaultHubCallTimeout
161+
httpClient.RetryMax = 5
158162
return &flistModule{
159163
root: root,
160164
flist: filepath.Join(root, "flist"),
@@ -168,9 +172,7 @@ func newFlister(root string, storage volumeAllocator, commander commander, syste
168172
commander: commander,
169173
system: system,
170174

171-
httpClient: &http.Client{
172-
Timeout: defaultHubCallTimeout,
173-
},
175+
httpClient: httpClient,
174176
}
175177
}
176178

@@ -731,14 +733,14 @@ func (f *flistModule) downloadInNamespace(name, u string) (resp *http.Response,
731733
return errors.Wrap(err, "failed to start tcp connection")
732734
}
733735

734-
cl := http.Client{
735-
Transport: &http.Transport{
736-
DisableKeepAlives: true,
737-
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
738-
return con, nil
739-
},
736+
cl := retryablehttp.NewClient()
737+
cl.HTTPClient.Transport = &http.Transport{
738+
DisableKeepAlives: true,
739+
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
740+
return con, nil
740741
},
741742
}
743+
cl.RetryMax = 5
742744

743745
resp, err = cl.Get(u)
744746
return err

‎pkg/gateway/metrics.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"strings"
1313

1414
"github.com/containernetworking/plugins/pkg/ns"
15+
"github.com/hashicorp/go-retryablehttp"
1516
"github.com/pkg/errors"
1617
"github.com/rs/zerolog/log"
1718
"github.com/threefoldtech/zos/pkg"
@@ -139,14 +140,14 @@ func metrics(rawUrl string) (map[string]*metric, error) {
139140

140141
defer con.Close()
141142

142-
cl := http.Client{
143-
Transport: &http.Transport{
144-
DisableKeepAlives: true,
145-
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
146-
return con, nil
147-
},
143+
cl := retryablehttp.NewClient()
144+
cl.HTTPClient.Transport = &http.Transport{
145+
DisableKeepAlives: true,
146+
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
147+
return con, nil
148148
},
149149
}
150+
cl.RetryMax = 5
150151

151152
response, err := cl.Get(rawUrl)
152153

‎pkg/geoip/geoip.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ package geoip
33
import (
44
"encoding/json"
55
"errors"
6-
"net/http"
76
"time"
87

8+
"github.com/hashicorp/go-retryablehttp"
99
"github.com/rs/zerolog/log"
1010
)
1111

@@ -22,9 +22,7 @@ type Location struct {
2222
var (
2323
geoipURLs = []string{"https://geoip.grid.tf/", "https://02.geoip.grid.tf/", "https://03.geoip.grid.tf/"}
2424

25-
defaultHTTPClient = &http.Client{
26-
Timeout: 10 * time.Second,
27-
}
25+
defaultHTTPClient = retryablehttp.NewClient()
2826
)
2927

3028
// Fetch retrieves the location of the system calling this function
@@ -51,6 +49,8 @@ func getLocation(geoIPService string) (Location, error) {
5149
City: "Unknown",
5250
}
5351

52+
defaultHTTPClient.HTTPClient.Timeout = 10 * time.Second
53+
defaultHTTPClient.RetryMax = 5
5454
resp, err := defaultHTTPClient.Get(geoIPService)
5555
if err != nil {
5656
return l, err

‎pkg/perf/publicip/publicip_task.go‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"github.com/cenkalti/backoff/v3"
1414
"github.com/containernetworking/plugins/pkg/ns"
15+
"github.com/hashicorp/go-retryablehttp"
1516
"github.com/rs/zerolog/log"
1617
substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
1718
"github.com/threefoldtech/zos/pkg/environment"
@@ -271,14 +272,15 @@ func getRealPublicIP() (net.IP, error) {
271272

272273
defer con.Close()
273274

274-
cl := http.Client{
275-
Transport: &http.Transport{
276-
DisableKeepAlives: true,
277-
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
278-
return con, nil
279-
},
275+
cl := retryablehttp.NewClient()
276+
cl.HTTPClient.Transport = &http.Transport{
277+
DisableKeepAlives: true,
278+
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
279+
return con, nil
280280
},
281281
}
282+
cl.RetryMax = 5
283+
282284
response, err := cl.Get("https://api.ipify.org/")
283285
if err != nil {
284286
return nil, errors.Join(err, errPublicIPLookup)

‎pkg/provision/engine.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"time"
1414

1515
"github.com/cenkalti/backoff/v3"
16+
"github.com/hashicorp/go-retryablehttp"
1617
"github.com/joncrlsn/dque"
1718
"github.com/pkg/errors"
1819
"github.com/rs/zerolog/log"
@@ -91,6 +92,8 @@ const (
9192
opPause
9293
// opResume resumes a deployment
9394
opResume
95+
// servers default timeout
96+
defaultHttpTimeout = 10 * time.Second
9497
)
9598

9699
// engineJob is a persisted job instance that is
@@ -1210,11 +1213,11 @@ func isTwinVerified(twinID uint32) (verified bool, err error) {
12101213
q.Set("twin_id", fmt.Sprint(twinID))
12111214
request.URL.RawQuery = q.Encode()
12121215

1213-
cl := &http.Client{
1214-
Timeout: 10 * time.Second,
1215-
}
1216+
cl := retryablehttp.NewClient()
1217+
cl.HTTPClient.Timeout = defaultHttpTimeout
1218+
cl.RetryMax = 5
12161219

1217-
response, err := cl.Do(request)
1220+
response, err := cl.StandardClient().Do(request)
12181221
if err != nil {
12191222
return
12201223
}

0 commit comments

Comments
 (0)