Storage hdd (#2265)

* WIP: enable using hdd for vms and other workloads

* implement pools policy

* disable hdd for workloads for now

Author: muhamadazmy

pkg/kernel/kernel.go

@@ -18,6 +18,10 @@ const (
 	VirtualMachine = "zos-debug-vm"
 	// if disable-gpu flag is provided gpu feature will be disabled on that node
 	DisableGPU = "disable-gpu"
+
+	// This allows the node to work without ssd disk. If ssd disk is available
+	// it will still be preferred for workloads. Otherwise fall back on HDD
+	MissingSSD = "missing-ssd"
 )
 
 // Params represent the parameters passed to the kernel at boot

pkg/storage/device.go

@@ -2,11 +2,13 @@ package storage
 
 import (
 	"fmt"
+	"slices"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg"
 	"github.com/threefoldtech/zos/pkg/gridtypes"
+	"github.com/threefoldtech/zos/pkg/storage/filesystem"
 )
 
 const (
@@ -17,7 +19,7 @@ const (
 func (m *Module) Devices() ([]pkg.Device, error) {
 	var devices []pkg.Device
 	log.Debug().Int("disks", len(m.hdds)).Msg("listing zdb devices")
-	for _, hdd := range m.hdds {
+	for _, hdd := range m.pools(PolicyHDDOnly) {
 		log.Debug().Str("device", hdd.Path()).Msg("checking device")
 		if _, err := hdd.Mounted(); err != nil {
 			log.Debug().Str("device", hdd.Path()).Msg("not mounted")
@@ -29,10 +31,12 @@ func (m *Module) Devices() ([]pkg.Device, error) {
 			log.Error().Err(err).Str("pool", hdd.Name()).Msg("failed to get pool volumes")
 			continue
 		}
+
 		usage, err := hdd.Usage()
 		if err != nil {
 			return nil, err
 		}
+
 		for _, vol := range volumes {
 			log.Debug().Str("volume", vol.Path()).Str("name", vol.Name()).Msg("checking volume")
 			if vol.Name() != zdbVolume {
@@ -56,7 +60,7 @@ func (m *Module) Devices() ([]pkg.Device, error) {
 
 // DeviceLookup looks up device by name
 func (m *Module) DeviceLookup(name string) (pkg.Device, error) {
-	for _, hdd := range m.hdds {
+	for _, hdd := range m.pools(PolicyHDDOnly) {
 		if hdd.Name() != name {
 			continue
 		}
@@ -98,12 +102,7 @@ func (m *Module) DeviceLookup(name string) (pkg.Device, error) {
 // DeviceAllocate allocates a new free device, allocation is done
 // by creation a zdb subvolume
 func (m *Module) DeviceAllocate(min gridtypes.Unit) (pkg.Device, error) {
-	for _, hdd := range m.hdds {
-		if _, err := hdd.Mounted(); err == nil {
-			// mounted pool. skip
-			continue
-		}
-
+	for _, hdd := range m.pools(PolicyHDDOnly) {
 		if hdd.Device().Size < uint64(min) {
 			continue
 		}
@@ -119,20 +118,29 @@ func (m *Module) DeviceAllocate(min gridtypes.Unit) (pkg.Device, error) {
 			continue
 		}
 
-		if len(volumes) != 0 {
-			log.Info().Str("pool", hdd.Name()).Msg("pool is already used")
+		exists := slices.ContainsFunc(volumes, func(vol filesystem.Volume) bool {
+			return vol.Name() == zdbVolume
+		})
+
+		if exists {
+			// a zdb volume already existws
 			continue
 		}
 
-		volume, err := hdd.AddVolume(zdbVolume)
+		usage, err := hdd.Usage()
 		if err != nil {
-			log.Error().Err(err).Msg("failed to create zdb volume")
+			return pkg.Device{}, err
+		}
+
+		if usage.Used+uint64(min) > usage.Size {
+			// not enough space
 			continue
 		}
 
-		usage, err := hdd.Usage()
+		volume, err := hdd.AddVolume(zdbVolume)
 		if err != nil {
-			return pkg.Device{}, err
+			log.Error().Err(err).Msg("failed to create zdb volume")
+			continue
 		}
 
 		return pkg.Device{

pkg/storage/disk.go

@@ -25,7 +25,7 @@ const (
 func (s *Module) diskPools() ([]string, error) {
 
 	var paths []string
-	for _, pool := range s.ssds {
+	for _, pool := range s.pools(PolicySSDFirst) {
 		if _, err := pool.Mounted(); err != nil {
 			continue
 		}
@@ -45,9 +45,9 @@ func (s *Module) diskPools() ([]string, error) {
 	return paths, nil
 }
 
-// VDiskFindCandidate find a suitbale location for creating a vdisk of the given size
+// VDiskFindCandidate find a suitable location for creating a vdisk of the given size
 func (s *Module) diskFindCandidate(size gridtypes.Unit) (path string, err error) {
-	candidates, err := s.findCandidates(size)
+	candidates, err := s.findCandidates(size, PolicySSDFirst)
 	if err != nil {
 		return path, err
 	}
@@ -78,13 +78,13 @@ func (s *Module) diskFindCandidate(size gridtypes.Unit) (path string, err error)
 }
 
 func (s *Module) findDisk(id string) (string, error) {
-	pools, err := s.diskPools()
+	vdiskVolumes, err := s.diskPools()
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to find disk with id '%s'", id)
 	}
 
-	for _, pool := range pools {
-		path, err := s.safePath(pool, id)
+	for _, volumePath := range vdiskVolumes {
+		path, err := s.safePath(volumePath, id)
 		if err != nil {
 			return "", err
 		}

pkg/storage/pools.go

@@ -0,0 +1,61 @@
+package storage
+
+import (
+	"slices"
+
+	"github.com/threefoldtech/zos/pkg/kernel"
+	"github.com/threefoldtech/zos/pkg/storage/filesystem"
+)
+
+// utils for pool ordering and presence
+
+// Policy needed
+type Policy func(s *Module) []filesystem.Pool
+
+func poolCmp(a, b filesystem.Pool) int {
+	_, errA := a.Mounted()
+	_, errB := b.Mounted()
+
+	// if the 2 pools has the same mount state, then both can
+	// be 0
+	if errA != nil && errB != nil || errA == nil && errB == nil {
+		return 0
+	} else if errA == nil {
+		// mounted pool comes first
+		return -1
+	} else {
+		return 1
+	}
+}
+
+func PolicySSDOnly(s *Module) []filesystem.Pool {
+	slices.SortFunc(s.ssds, poolCmp)
+	return s.ssds
+}
+
+func PolicyHDDOnly(s *Module) []filesystem.Pool {
+	slices.SortFunc(s.hdds, poolCmp)
+	return s.hdds
+}
+
+func PolicySSDFirst(s *Module) []filesystem.Pool {
+	pools := PolicySSDOnly(s)
+
+	// if missing ssd is supported, this policy
+	// will also use the hdd pools for provisioning
+	// and cache
+	//
+	// TODO: this is fully disabled now. and a ssd first policy
+	// will act like a ssd-only policy.
+	// to enable workloads on hdd, drop the false part.
+	if false && kernel.GetParams().Exists(kernel.MissingSSD) {
+		pools = append(pools, PolicyHDDOnly(s)...)
+	}
+
+	return pools
+}
+
+// get available pools in defined presence
+func (s *Module) pools(policy Policy) []filesystem.Pool {
+	return policy(s)
+}