Merge remote-tracking branch 'origin/cohere' into sync/upstream-2026-04-16-a2c24858

Author: yousef-cohere

.github/workflows/peerpods-chart_image.yaml

@@ -155,6 +155,25 @@ jobs:
             --password-stdin
           echo "Helm authenticated with ghcr.io"
 
+      - name: Authenticate to GCP
+        if: ${{ contains(steps.registry.outputs.registry, 'docker.pkg.dev') }}
+        uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.13
+        with:
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.GCP_SERVICE_ACCOUNT }}
+
+      - name: Authenticate Helm with Artifact Registry
+        if: ${{ contains(steps.registry.outputs.registry, 'docker.pkg.dev') }}
+        env:
+          REGISTRY: ${{ steps.registry.outputs.registry }}
+        run: |
+          AR_HOST=$(echo "${REGISTRY}" | cut -d'/' -f1)
+          echo "Authenticating Helm with ${AR_HOST}..."
+          gcloud auth print-access-token | helm registry login "${AR_HOST}" \
+            --username oauth2accesstoken \
+            --password-stdin
+          echo "Helm authenticated with ${AR_HOST}"
+
       - name: Update Helm dependencies
         run: |
           echo "Updating Helm dependencies..."

src/cloud-api-adaptor/install/charts/peerpods/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: peerpods
 description: Cloud API Adaptor (Peerpods) Helm Chart
 type: application
-version: 0.1.4
+version: 0.1.4-cohere.1
 appVersion: "v0.19.0"
 
 keywords:

src/cloud-api-adaptor/install/charts/peerpods/providers/gcp.yaml

@@ -66,6 +66,10 @@ providerConfigs:
     # (default: "")
     # GCP_SUBNETWORK: ""
 
+    # Use Spot (preemptible) instances for peer pod VMs. Reduces cost significantly but VMs may be reclaimed by GCP at any time.
+    # (default: "false")
+    # GCP_USE_SPOT_INSTANCES: "false"
+
     # Zone
     # (required)
     GCP_ZONE: ""

src/cloud-api-adaptor/install/charts/peerpods/templates/daemonset.yaml

@@ -25,7 +25,11 @@ spec:
       serviceAccountName: cloud-api-adaptor
       containers:
       - command:
+        {{- if .Values.command }}
+        {{- toYaml .Values.command | nindent 8 }}
+        {{- else }}
         - /usr/local/bin/entrypoint.sh
+        {{- end }}
         env:
         - name: NODE_NAME
           valueFrom:
@@ -100,7 +104,7 @@ spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
       nodeSelector:
-        node.kubernetes.io/worker: ""
+        {{- toYaml .Values.nodeSelector | nindent 8 }}
       volumes:
 {{- if and (or (eq .Values.provider "libvirt") (eq .Values.provider "byom")) (include "peerpods.sshKeySecretName" .) }}
       - name: ssh

src/cloud-api-adaptor/install/charts/peerpods/templates/fix-gke-node-config.yaml

@@ -0,0 +1,94 @@
+{{- if .Values.gkeNodeFix.enabled }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fix-gke-node-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: fix-gke-node-config
+spec:
+  selector:
+    matchLabels:
+      app: fix-gke-node-config
+  template:
+    metadata:
+      labels:
+        app: fix-gke-node-config
+    spec:
+      tolerations:
+      - operator: Exists
+      hostPID: true
+      nodeSelector:
+        {{- toYaml .Values.nodeSelector | nindent 8 }}
+      initContainers:
+      - name: fix-containerd
+        image: alpine:3.21
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: host-containerd-config
+          mountPath: /etc/containerd
+        command:
+        - /bin/sh
+        - -c
+        - |
+          TOML=/etc/containerd/config.toml
+
+          if grep -q 'discard_unpacked_layers = false' "$TOML"; then
+            echo "containerd: already patched, skipping"
+            exit 0
+          fi
+
+          if grep -q 'discard_unpacked_layers = true' "$TOML"; then
+            sed -i 's/discard_unpacked_layers = true/discard_unpacked_layers = false/g' "$TOML"
+            echo "containerd: patched $TOML"
+            grep discard_unpacked_layers "$TOML"
+            nsenter -t 1 -m -u -i -n -p -- systemctl restart containerd
+            echo "containerd: restarted"
+          else
+            echo "containerd: discard_unpacked_layers setting not found, skipping restart"
+            grep discard_unpacked_layers "$TOML" || true
+          fi
+      - name: fix-kubelet
+        image: alpine:3.21
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: host-kubelet-config
+          mountPath: /home/kubernetes
+        command:
+        - /bin/sh
+        - -c
+        - |
+          CONFIG=/home/kubernetes/kubelet-config.yaml
+          DESIRED="{{ .Values.gkeNodeFix.runtimeRequestTimeout }}"
+
+          if grep -q "runtimeRequestTimeout: \"$DESIRED\"" "$CONFIG" 2>/dev/null; then
+            echo "kubelet: already set to $DESIRED, skipping"
+            exit 0
+          fi
+
+          if grep -q "runtimeRequestTimeout:" "$CONFIG"; then
+            sed -i "s/runtimeRequestTimeout:.*/runtimeRequestTimeout: \"$DESIRED\"/" "$CONFIG"
+          else
+            echo "runtimeRequestTimeout: \"$DESIRED\"" >> "$CONFIG"
+          fi
+
+          echo "kubelet: patched $CONFIG"
+          grep runtimeRequestTimeout "$CONFIG"
+
+          nsenter -t 1 -m -u -i -n -p -- systemctl restart kubelet
+          echo "kubelet: restarted"
+      containers:
+      - name: pause
+        image: registry.k8s.io/pause:3.10
+      volumes:
+      - name: host-containerd-config
+        hostPath:
+          path: /etc/containerd
+          type: Directory
+      - name: host-kubelet-config
+        hostPath:
+          path: /home/kubernetes
+          type: Directory
+{{- end }}

src/cloud-api-adaptor/install/charts/peerpods/templates/rbac.yaml

@@ -5,6 +5,10 @@ kind: ServiceAccount
 metadata:
   name: cloud-api-adaptor
   namespace: {{ .Release.Namespace }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

src/cloud-api-adaptor/install/charts/peerpods/values.yaml

@@ -79,11 +79,29 @@ secrets:
 #
 # Note: libvirt/docker provider files override this with the dev image.
 image:
-  name: quay.io/confidential-containers/cloud-api-adaptor
-  tag: "latest"
+  name: us-central1-docker.pkg.dev/cohere-confidential-computing/cloud-api-adaptor/cloud-api-adaptor
+  tag: "44d97551e74d3d835e6ab17d37475f1033b2011a"
 
 # Cloud provider: libvirt, aws, azure, gcp, ibmcloud, vsphere
-provider: libvirt
+provider: gcp
+
+# Override the DaemonSet container command. Set to bypass entrypoint.sh when
+# using Workload Identity (no GCP_CREDENTIALS file needed).
+command: ["/bin/sh", "-c", "exec cloud-api-adaptor gcp"]
+
+# DaemonSet nodeSelector. GKE nodes are labeled by the node pool config.
+nodeSelector:
+  fortress.cohere.com/caa-worker: "true"
+
+# ServiceAccount annotations (e.g. Workload Identity binding)
+serviceAccount:
+  annotations: {}
+
+# GKE node config fixes. Deploys a separate DaemonSet that patches containerd
+# (discard_unpacked_layers) and kubelet (runtimeRequestTimeout) on each node.
+gkeNodeFix:
+  enabled: true
+  runtimeRequestTimeout: "15m"
 
 # Maximum number of peer pods allowed to run simultaneously
 # This limit prevents resource exhaustion on the cloud provider
@@ -117,6 +135,9 @@ daemonset:
 # Configuration options documented in ../../../peerpod-ctrl/chart/values.yaml
 resourceCtrl:
   enabled: true
+  image:
+    repository: us-central1-docker.pkg.dev/cohere-confidential-computing/cloud-api-adaptor/peerpod-ctrl
+    tag: "bfd5b9847b2fb44c843f6a3b6209092d0a217d83"
 
 # peerpods-webhook subchart configuration
 # Mutating webhook that modifies pod specs to use peer pods runtime and resources

src/cloud-api-adaptor/pkg/util/agentproto/redirector.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net"
 	"sync"
+	"time"
 
 	"github.com/containerd/ttrpc"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
@@ -17,6 +18,12 @@ import (
 	pb "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
 )
 
+// statsTimeout bounds how long StatsContainer and GetMetrics RPCs may block.
+// Keeping this well under the metrics-server scrape interval (~10s) prevents a
+// single slow or unreachable peer-pod VM from stalling kubelet stats collection
+// for the entire node.
+const statsTimeout = 5 * time.Second
+
 type Redirector interface {
 	pb.AgentServiceService
 	pb.HealthService
@@ -154,6 +161,8 @@ func (s *redirector) StatsContainer(ctx context.Context, req *pb.StatsContainerR
 	if err := s.Connect(ctx); err != nil {
 		return nil, err
 	}
+	ctx, cancel := context.WithTimeout(ctx, statsTimeout)
+	defer cancel()
 	return s.agentClient.StatsContainer(ctx, req)
 }
 
@@ -281,6 +290,8 @@ func (s *redirector) GetMetrics(ctx context.Context, req *pb.GetMetricsRequest)
 	if err := s.Connect(ctx); err != nil {
 		return nil, err
 	}
+	ctx, cancel := context.WithTimeout(ctx, statsTimeout)
+	defer cancel()
 	return s.agentClient.GetMetrics(ctx, req)
 }
 
@@ -393,6 +404,8 @@ func (s *redirector) GetVolumeStats(ctx context.Context, req *pb.VolumeStatsRequ
 	if err := s.Connect(ctx); err != nil {
 		return nil, err
 	}
+	ctx, cancel := context.WithTimeout(ctx, statsTimeout)
+	defer cancel()
 	return s.agentClient.GetVolumeStats(ctx, req)
 }
 

src/cloud-api-adaptor/podvm-mkosi/Dockerfile.mkosi

@@ -33,6 +33,7 @@ COPY mkosi.presets /image/mkosi.presets
 COPY mkosi.profiles /image/mkosi.profiles
 COPY mkosi.skeleton /image/mkosi.skeleton
 COPY mkosi.skeleton-debug /image/mkosi.skeleton-debug
+COPY mkosi.skeleton-debug-fedora /image/mkosi.skeleton-debug-fedora
 COPY mkosi.skeleton-sftp /image/mkosi.skeleton-sftp
 COPY mkosi.workspace /image/mkosi.workspace
 COPY resources /image/resources

src/cloud-api-adaptor/podvm-mkosi/Dockerfile.mkosi.ubuntu

@@ -1,17 +1,23 @@
 # syntax=docker/dockerfile:1.5.0-labs
-# ubuntu:24.04
-FROM ubuntu@sha256:0d39fcc8335d6d74d5502f6df2d30119ff4790ebbb60b364818d5112d9e3e932 AS builder
+# ubuntu:24.10 (Oracular) — provides systemd 256 EFI stub with TDX RTMR support
+FROM ubuntu@sha256:cdf755952ed117f6126ff4e65810bf93767d4c38f5c7185b50ec1f1078b464cc AS builder
 
 ARG MKOSI_VERSION="v22"
 ARG PROFILE="debug"
 ARG IMAGE_VERSION="0.0.0"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
+# 24.10 is EOL; redirect apt to old-releases mirror
+RUN sed -i 's|archive.ubuntu.com|old-releases.ubuntu.com|g; s|security.ubuntu.com|old-releases.ubuntu.com|g' \
+	/etc/apt/sources.list.d/ubuntu.sources
+
 RUN apt-get update && \
 	apt-get install -y \
 	bubblewrap \
+	curl \
 	git \
+	gnupg \
 	cpio \
 	systemd-repart \
 	kmod \
@@ -37,10 +43,39 @@ COPY mkosi.presets /image/mkosi.presets
 COPY mkosi.profiles /image/mkosi.profiles
 COPY mkosi.skeleton /image/mkosi.skeleton
 COPY mkosi.skeleton-debug /image/mkosi.skeleton-debug
+COPY mkosi.skeleton-debug-ubuntu /image/mkosi.skeleton-debug-ubuntu
 COPY mkosi.skeleton-sftp /image/mkosi.skeleton-sftp
 COPY mkosi.workspace /image/mkosi.workspace
 COPY resources /image/resources
 COPY mkosi.conf.ubuntu /image/mkosi.conf
+
+# Add NVIDIA APT repos to mkosi.skeleton/ so they are present in the image tree
+# *before* package installation. mkosi applies SkeletonTrees before apt-get runs.
+RUN mkdir -p /image/mkosi.skeleton/etc/apt/sources.list.d \
+             /image/mkosi.skeleton/etc/apt/preferences.d \
+             /image/mkosi.skeleton/usr/share/keyrings \
+    && curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-archive-keyring.gpg \
+       -o /image/mkosi.skeleton/usr/share/keyrings/cuda-archive-keyring.gpg \
+    && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
+       | gpg --dearmor -o /image/mkosi.skeleton/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
+    && echo 'deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/ /' \
+       > /image/mkosi.skeleton/etc/apt/sources.list.d/cuda-ubuntu2404-x86_64.list \
+    && echo 'deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://nvidia.github.io/libnvidia-container/stable/deb/amd64 /' \
+       > /image/mkosi.skeleton/etc/apt/sources.list.d/nvidia-container-toolkit.list \
+    && printf '%s\n' \
+       'Package: nvidia-*' \
+       'Pin: origin developer.download.nvidia.com' \
+       'Pin-Priority: 1001' \
+       '' \
+       'Package: libnvidia-*' \
+       'Pin: origin developer.download.nvidia.com' \
+       'Pin-Priority: 1001' \
+       '' \
+       'Package: cuda-*' \
+       'Pin: origin developer.download.nvidia.com' \
+       'Pin-Priority: 1001' \
+       > /image/mkosi.skeleton/etc/apt/preferences.d/nvidia-cuda-repo
+
 RUN --security=insecure mkosi --profile=$PROFILE --image-version=$IMAGE_VERSION
 
 FROM scratch