Skip to content

Commit ebfab30

Browse files
Nabil-Salahashraffouda
Nabil-Salah
authored and
ashraffouda
committed
feat: add booting with gpu scripts and update development docs
Signed-off-by: nabil salah <nabil.salah203@gmail.com>
1 parent 1469ca8 commit ebfab30

5 files changed

Lines changed: 200 additions & 1 deletion

File tree

‎qemu/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,7 @@ net:
2727
run:
2828
@echo "Running your node"
2929
sudo ./vm.sh -g -n node-01 -c "farmer_id=$(id) version=v3 printk.devmsg=on runmode=dev nomodeset ssh-user=$(user)"
30+
31+
run-gpu:
32+
@echo "Running your node"
33+
sudo ./vm_gpu.sh -g -n node-01 -c "farmer_id=$(id) version=v3 printk.devmsg=on runmode=dev nomodeset ssh-user=$(user)"

‎qemu/README.md‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Run 0-OS in a VM using qemu
22

3-
| For a quick development docs check [here](../docs/development/README.md)
3+
| For a quick development docs check [here](https://github.com/threefoldtech/zosbase/blob/main/docs/development/quickstart.md)
44

55
This folder contains a script that you can use to run 0-OS in a VM using qemu.
66

@@ -169,3 +169,21 @@ Note: `double quotes around the flags after -c are very important`
169169
## inspecting the cmdline Arguments
170170

171171
can be done using `cat /proc/cmdline`
172+
173+
## To boot gpu node
174+
175+
### check your gpus
176+
177+
```lspci | grep -i vga``` you will find each gpu start with pci.ids like "01:00.0"
178+
179+
It will print your available gpus you should have at least one gpu that will not be passed to the qemu vm so you can still use in your current machine because gpu that will be passed will be unlinked from your current device.
180+
181+
to unlink gpus you can simply run ```./setup-vfio.sh``` script but after updating it with your pci.ids also it works only with nvidia gpus **require to restart machine after**
182+
183+
to revert the unlinking run ```./revert-vfio.sh``` **require to restart machine after**
184+
185+
### now run your vm
186+
187+
update vfio-pci with your pci.ids in ```qemu-system-x86_64```
188+
189+
```make run-gpu id=<your_farm_id> user=<your_github_username>```

‎qemu/revert-vfio.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
echo "[1/5] Reverting GRUB parameters..."
6+
7+
# Clean VFIO-related parameters from GRUB_CMDLINE_LINUX_DEFAULT
8+
sudo sed -i 's/\s*vfio-pci.ids=[^" ]*//g' /etc/default/grub
9+
sudo sed -i 's/\s*intel_iommu=on//g' /etc/default/grub
10+
sudo sed -i 's/\s*iommu=pt//g' /etc/default/grub
11+
12+
echo "[2/5] Removing VFIO initramfs modules..."
13+
sudo rm -f /etc/initramfs-tools/modules
14+
15+
echo "[3/5] Removing NVIDIA blacklist..."
16+
sudo rm -f /etc/modprobe.d/blacklist-nvidia.conf
17+
18+
echo "[4/5] Regenerating initramfs..."
19+
sudo update-initramfs -u
20+
21+
echo "[5/5] Updating GRUB..."
22+
sudo update-grub
23+
24+
echo "✅ Reverted VFIO GPU passthrough configuration. Reboot to re-enable NVIDIA on host."

‎qemu/setup-vfio.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
echo "[1/5] Updating GRUB boot parameters..."
6+
7+
# Backup and update GRUB
8+
# Update vfio-pci.ids with your GPU IDs
9+
sudo sed -i.bak '/^GRUB_CMDLINE_LINUX_DEFAULT=/ s/"$/ intel_iommu=on iommu=pt vfio-pci.ids=10de:2560,10de:228e"/' /etc/default/grub
10+
11+
echo "[2/5] Creating vfio.conf for initramfs..."
12+
echo -e "vfio\nvfio_iommu_type1\nvfio_pci\nvfio_virqfd" | sudo tee /etc/initramfs-tools/modules
13+
14+
echo "[3/5] Blacklisting NVIDIA drivers..."
15+
cat <<EOF | sudo tee /etc/modprobe.d/blacklist-nvidia.conf
16+
blacklist nouveau
17+
blacklist nvidia
18+
blacklist nvidia_drm
19+
blacklist nvidia_modeset
20+
blacklist nvidia_uvm
21+
EOF
22+
23+
echo "[4/5] Regenerating initramfs..."
24+
sudo update-initramfs -u
25+
26+
echo "[5/5] Updating GRUB..."
27+
sudo update-grub
28+
29+
echo "✅ All done. Please reboot your system for changes to take effect."

‎qemu/vm_gpu.sh‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#!/bin/bash
2+
3+
4+
debug=0
5+
reset=0
6+
image=zos.efi
7+
kernelargs=""
8+
bridge=zos0
9+
graphics="-nographic -nodefaults"
10+
smp=1
11+
mem=3
12+
tpm=0
13+
14+
usage() {
15+
cat <<EOF
16+
Usage: vm -n $name [ -r ] [ -d ]
17+
-n $name: name of the vm
18+
-d: run in debug mode
19+
-r: reset vm (recreate disks)
20+
-c: Kernel command
21+
-s: number of virtual cpus
22+
-i: kernel image to boot
23+
-b: bridge for network (default zos)
24+
-g: open GUI
25+
-m: memory in Gigabytes
26+
-t: tpm support (requires swtpm)
27+
-h: help
28+
EOF
29+
exit 0
30+
}
31+
32+
33+
while getopts "c:n:i:rdtb:gs:m:" opt; do
34+
case $opt in
35+
i ) image=$OPTARG ;;
36+
r ) reset=1 ;;
37+
d ) debug=1 ;;
38+
n ) name=$OPTARG ;;
39+
c ) kernelargs=$OPTARG ;;
40+
b ) bridge=$OPTARG ;;
41+
g ) graphics="" ;;
42+
s ) smp=$OPTARG ;;
43+
m ) mem=$OPTARG ;;
44+
t ) tpm=1 ;;
45+
h ) usage ; exit 0 ;;
46+
\?) usage ; exit 1 ;;
47+
esac
48+
done
49+
shift $(($OPTIND - 1))
50+
51+
52+
cmdline="console=ttyS1,115200n8 zos-debug zos-debug-vm $kernelargs"
53+
basepath=$(dirname $0)
54+
vmdir="$basepath/$name"
55+
md5=$(echo -n $name | md5sum | awk '{print $1}')
56+
uuid="${md5:0:8}-${md5:8:4}-${md5:12:4}-${md5:16:4}-${md5:20}"
57+
basemac="54:${md5:2:2}:${md5:4:2}:${md5:6:2}:${md5:8:2}:${md5:10:1}"
58+
59+
createdisks() {
60+
qemu-img create -f qcow2 $vmdir/vda.qcow2 500G
61+
qemu-img create -f qcow2 $vmdir/vdb.qcow2 500G
62+
qemu-img create -f qcow2 $vmdir/vdc.qcow2 500G
63+
qemu-img create -f qcow2 $vmdir/vdd.qcow2 500G
64+
qemu-img create -f qcow2 $vmdir/vde.qcow2 500G
65+
66+
}
67+
68+
if [[ ! -d "$vmdir" || "$reset" -eq 1 ]]; then
69+
mkdir -p "$vmdir"
70+
createdisks
71+
fi
72+
73+
if ps -eaf | grep -v grep | grep "$uuid" > /dev/null; then
74+
echo "VM $name is already running"
75+
exit 1
76+
fi
77+
78+
tpmargs=""
79+
if [[ $tpm -eq "1" ]]; then
80+
if ! command -v swtpm &> /dev/null; then
81+
echo "tpm option require `swtpm` please install first"
82+
exit 1
83+
fi
84+
pkill swtpm
85+
tpm_dir="$vmdir/tpm"
86+
tpm_socket="$vmdir/swtpm.sock"
87+
mkdir -p $tpm_dir
88+
rm $tpm_socket &> /dev/null || true
89+
# runs in the backgroun
90+
swtpm \
91+
socket --tpm2 \
92+
--tpmstate dir=$tpm_dir \
93+
--ctrl type=unixio,path=$vmdir/swtpm.sock \
94+
--log level=20 &> tpm.logs &
95+
96+
while [ ! -S "$tpm_socket" ]; do
97+
echo "waiting for tpm"
98+
sleep 1s
99+
done
100+
sleep 1s
101+
tpmargs="-chardev socket,id=chrtpm,path=${tpm_socket} -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis,tpmdev=tpm0"
102+
fi
103+
104+
echo "boot $image"
105+
106+
qemu-system-x86_64 -kernel $image \
107+
-m $(( mem * 1024 )) \
108+
-enable-kvm \
109+
-cpu host,host-phys-bits \
110+
-smp $smp \
111+
-uuid $uuid \
112+
-netdev bridge,id=zos0,br=${bridge} -device virtio-net-pci,netdev=zos0,mac="${basemac}1" \
113+
-drive file=fat:rw:$basepath/overlay,format=raw \
114+
-append "${cmdline}" \
115+
-drive file=$vmdir/vda.qcow2,if=virtio -drive file=$vmdir/vdb.qcow2,if=virtio \
116+
-drive file=$vmdir/vdc.qcow2,if=virtio -drive file=$vmdir/vdd.qcow2,if=virtio \
117+
-drive file=$vmdir/vde.qcow2,if=virtio \
118+
-serial null -serial mon:stdio \
119+
${graphics} \
120+
${tpmargs} \
121+
-device pci-bridge,chassis_nr=1,id=pcie.1 \
122+
-device vfio-pci,host=01:00.0,bus=pcie.1,addr=00.0,multifunction=on \
123+
-device vfio-pci,host=01:00.1,bus=pcie.1,addr=00.1
124+
;

0 commit comments

Comments
 (0)