boot-utils: Download rootfs images from GitHub releases

Now that rootfs images will be uploaded to GitHub releases, both
boot-qemu.py and boot-uml.py need to download the images before running.

The latest rootfs release can be queried from GitHub's API, which
returns JSON with information about the release tag and assets to allow
easy downloading. Unfortunately, GitHub's API has a rate limit, so
always querying is not possible. To account for this, the script will
first query the rate_limit endpoint to see how many queries are
remaining for the current user. If there are queries remaining, the
script will download the rootfs image if it does not already exist or if
the tag in the '.release' file is different from the latest on. If
there are no queries remaining and the image does not already exist, the
script errors and instructs the user how to make an authenticated API
request or manually download the images themselves from the web.

Signed-off-by: Nathan Chancellor <nathan@kernel.org>

Author: nathanchance

boot-qemu.py

@@ -14,7 +14,6 @@
 
 import utils
 
-BOOT_UTILS = Path(__file__).resolve().parent
 SUPPORTED_ARCHES = [
     'arm',
     'arm32_v5',
@@ -163,16 +162,7 @@ def _have_dev_kvm_access(self):
     def _prepare_initrd(self):
         if not self._initrd_arch:
             raise RuntimeError('No initrd architecture specified?')
-        if not (src := Path(BOOT_UTILS, 'images', self._initrd_arch,
-                            'rootfs.cpio.zst')).exists():
-            raise FileNotFoundError(f"initrd ('{src}') does not exist?")
-
-        (dst := src.with_suffix('')).unlink(missing_ok=True)
-
-        utils.check_cmd('zstd')
-        subprocess.run(['zstd', '-d', src, '-o', dst, '-q'], check=True)
-
-        return dst
+        return utils.prepare_initrd(self._initrd_arch)
 
     def _run_fg(self):
         # Pretty print and run QEMU command
@@ -370,7 +360,7 @@ def _can_use_kvm(self):
         # 32-bit EL1 is not supported on all cores so support for it must be
         # explicitly queried via the KVM_CHECK_EXTENSION ioctl().
         try:
-            subprocess.run(Path(BOOT_UTILS, 'utils',
+            subprocess.run(Path(utils.BOOT_UTILS, 'utils',
                                 'aarch64_32_bit_el1_supported'),
                            check=True)
         except subprocess.CalledProcessError:
@@ -437,7 +427,7 @@ def _setup_efi(self):
         ]
         aavmf = utils.find_first_file(usr_share, aavmf_locations)
 
-        self._efi_img = Path(BOOT_UTILS, 'images', self._initrd_arch,
+        self._efi_img = Path(utils.BOOT_UTILS, 'images', self._initrd_arch,
                              'efi.img')
         # This file is in /usr/share, so it must be copied in order to be
         # modified.
@@ -652,7 +642,7 @@ def run(self):
                 Path('OVMF/OVMF_VARS.fd'),  # Debian and Ubuntu
             ]
             ovmf_vars = utils.find_first_file(usr_share, ovmf_vars_locations)
-            self._efi_vars = Path(BOOT_UTILS, 'images', self.initrd_arch,
+            self._efi_vars = Path(utils.BOOT_UTILS, 'images', self.initrd_arch,
                                   ovmf_vars.name)
             # This file is in /usr/share, so it must be copied in order to be
             # modified.

boot-uml.py

@@ -2,13 +2,10 @@
 # pylint: disable=invalid-name
 
 import argparse
-from pathlib import Path
 import subprocess
 
 import utils
 
-base_folder = Path(__file__).resolve().parent
-
 
 def parse_arguments():
     """
@@ -38,26 +35,6 @@ def parse_arguments():
     return parser.parse_args()
 
 
-def decomp_rootfs():
-    """
-    Decompress and get the full path of the initial ramdisk for use with UML.
-
-    Returns:
-        rootfs (Path): rootfs Path object containing full path to rootfs.
-    """
-    rootfs = base_folder.joinpath("images", "x86_64", "rootfs.ext4")
-
-    # This could be 'rootfs.unlink(missing_ok=True)' but that was only added in Python 3.8.
-    if rootfs.exists():
-        rootfs.unlink()
-
-    utils.check_cmd("zstd")
-    subprocess.run(["zstd", "-q", "-d", f"{rootfs}.zst", "-o", rootfs],
-                   check=True)
-
-    return rootfs
-
-
 def run_kernel(kernel_image, rootfs, interactive):
     """
     Run UML command with path to rootfs and additional arguments based on user
@@ -78,5 +55,6 @@ def run_kernel(kernel_image, rootfs, interactive):
 if __name__ == '__main__':
     args = parse_arguments()
     kernel = utils.get_full_kernel_path(args.kernel_location, "linux")
+    initrd = utils.prepare_initrd('x86_64', rootfs_format='ext4')
 
-    run_kernel(kernel, decomp_rootfs(), args.interactive)
+    run_kernel(kernel, initrd, args.interactive)

utils.py

@@ -1,9 +1,15 @@
 #!/usr/bin/env python3
 
+import json
+import os
 from pathlib import Path
+import subprocess
 import shutil
 import sys
 
+BOOT_UTILS = Path(__file__).resolve().parent
+REPO = 'ClangBuiltLinux/boot-utils'
+
 
 def check_cmd(cmd):
     """
@@ -30,6 +36,36 @@ def die(string):
     sys.exit(1)
 
 
+def download_initrd(gh_json, local_dest):
+    """
+    Download an initial ramdisk from a GitHub release
+
+    Parameters:
+        gh_json (dict): A serialized JSON object from a repo's release endpoint
+        local_dest (Path): A Path object pointing to the local file destination
+    """
+    assets = gh_json['assets']
+    tag = gh_json['tag_name']
+    url = gh_json['url']
+
+    # Turns '<arch>/rootfs.<format>.zst' into '<arch>-rootfs.<format>.zst'
+    remote_file = '-'.join(local_dest.parts[-2:])
+
+    for asset in assets:
+        if asset['name'] == remote_file:
+            curl_cmd = [
+                'curl', '-LSs', '-o', local_dest, asset['browser_download_url']
+            ]
+            subprocess.run(curl_cmd, check=True)
+
+            # Update the '.release' file in the same folder as the download
+            local_dest.with_name('.release').write_text(tag, encoding='utf-8')
+
+            return
+
+    raise RuntimeError(f"Failed to find {remote_file} in downloads of {url}?")
+
+
 def find_first_file(relative_root, possible_files, required=True):
     """
     Attempts to find the first option available in the list of files relative
@@ -93,6 +129,38 @@ def get_full_kernel_path(kernel_location, image, arch=None):
     return kernel.resolve()
 
 
+def get_gh_json(endpoint):
+    """
+    Query a GitHub API endpoint.
+
+    Parameters:
+        endpoint (str): The URL of the endpoint to query.
+
+    Returns:
+        A JSON object from the result of the query.
+    """
+    curl_cmd = ['curl', '-LSs']
+    if 'GITHUB_TOKEN' in os.environ:
+        curl_cmd += [
+            '-H',
+            'Accept: application/vnd.github+json',
+            '-H',
+            f"Authorization: Bearer {os.environ['GITHUB_TOKEN']}",
+        ]
+    curl_cmd.append(endpoint)
+
+    try:
+        curl_out = subprocess.run(curl_cmd,
+                                  capture_output=True,
+                                  check=True,
+                                  text=True).stdout
+    except subprocess.CalledProcessError as err:
+        raise RuntimeError(
+            f"Failed to query GitHub API at {endpoint}: {err.stderr}") from err
+
+    return json.loads(curl_out)
+
+
 def green(string):
     """
     Prints string in bold green.
@@ -103,6 +171,52 @@ def green(string):
     print(f"\n\033[01;32m{string}\033[0m", flush=True)
 
 
+def prepare_initrd(architecture, rootfs_format='cpio'):
+    """
+    Returns a decompressed initial ramdisk.
+
+    Parameters:
+        architecture (str): Architecture to download image for.
+        rootfs_format (str): Initrd format ('cpio' or 'ext4')
+    """
+    src = Path(BOOT_UTILS, 'images', architecture,
+               f"rootfs.{rootfs_format}.zst")
+    src.parent.mkdir(exist_ok=True, parents=True)
+
+    # First, make sure that the current user is not rate limited by GitHub,
+    # otherwise the next API call will not return valid information.
+    gh_json_rl = get_gh_json('https://api.github.com/rate_limit')
+    limit = gh_json_rl['resources']['core']['limit']
+    remaining = gh_json_rl['resources']['core']['remaining']
+
+    # If we have API calls remaining, we can query for the latest release to
+    # make sure that we are up to date.
+    if remaining > 0:
+        gh_json_rel = get_gh_json(
+            f"https://api.github.com/repos/{REPO}/releases/latest")
+        # Download the ramdisk if it is not already downloaded
+        if not src.exists():
+            download_initrd(gh_json_rel, src)
+        # If it is already downloaded, check that it is up to date and download
+        # an update only if necessary.
+        elif (rel_file := src.with_name('.release')).exists():
+            cur_rel = rel_file.read_text(encoding='utf-8')
+            latest_rel = gh_json_rel['tag_name']
+            if cur_rel != latest_rel:
+                download_initrd(gh_json_rel, src)
+    elif not src.exists():
+        raise RuntimeError(
+            f"Cannot query GitHub API for latest images release due to rate limit (remaining: {remaining}, limit: {limit}) and {src} does not exist already! "
+            'Download it manually or supply a GitHub personal access token via the GITHUB_TOKEN environment variable to make an authenticated GitHub API request.'
+        )
+
+    check_cmd('zstd')
+    (dst := src.with_suffix('')).unlink(missing_ok=True)
+    subprocess.run(['zstd', '-d', src, '-o', dst, '-q'], check=True)
+
+    return dst
+
+
 def red(string):
     """
     Prints string in bold red.