[SVLS-8740] add e2e test trigger (#1173)

## Overview
Trigger E2E test runs in this repo. This should also add E2E test runs
to the release pipeline, but does not block on failure in case of
flakiness.

## Testing
Pipeline launches E2E tests

Author: shreyamalpani

.gitlab/scripts/poll_e2e.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Trigger serverless-e2e-tests pipeline and poll until completion.
+
+set -euo pipefail
+
+if [ -z "${EXTENSION_LAYER_ARN:-}" ]; then
+    echo "ERROR: EXTENSION_LAYER_ARN is not set or empty"
+    exit 1
+fi
+
+E2E_PROJECT_ENCODED="DataDog%2Fserverless-e2e-tests"
+E2E_REF="${E2E_REF:-main}"
+
+curl -OL "binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" && mv authanywhere-linux-amd64 /bin/authanywhere && chmod +x /bin/authanywhere
+
+BTI_CI_API_TOKEN=$(authanywhere --audience rapid-devex-ci)
+
+BTI_RESPONSE=$(curl --silent --request GET \
+    --header "$BTI_CI_API_TOKEN" \
+    --header "Content-Type: application/vnd.api+json" \
+    "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=serverless-e2e-tests")
+
+GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty')
+if [ -z "$GITLAB_TOKEN" ]; then
+    echo "ERROR: could not obtain GitLab token from BTI"
+    exit 1
+fi
+
+echo "Triggering DataDog/serverless-e2e-tests pipeline (ref: ${E2E_REF})..."
+echo "  EXTENSION_LAYER_ARN=${EXTENSION_LAYER_ARN}"
+
+TRIGGER_RESPONSE=$(curl --silent --request POST \
+    --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \
+    --header "Content-Type: application/json" \
+    --data "$(jq -n \
+        --arg ref "$E2E_REF" \
+        --arg arn "$EXTENSION_LAYER_ARN" \
+        '{ref: $ref, variables: [{key: "EXTENSION_VERSION", value: $arn}]}')" \
+    "${CI_API_V4_URL}/projects/${E2E_PROJECT_ENCODED}/pipeline")
+
+PIPELINE_ID=$(echo "$TRIGGER_RESPONSE" | jq -r '.id // empty')
+PIPELINE_URL=$(echo "$TRIGGER_RESPONSE" | jq -r '.web_url // empty')
+
+if [ -z "$PIPELINE_ID" ] || [ "$PIPELINE_ID" = "null" ]; then
+    echo "ERROR: failed to trigger downstream pipeline"
+    echo "Response: $TRIGGER_RESPONSE"
+    exit 1
+fi
+
+echo "Triggered downstream pipeline: ${PIPELINE_URL} (ID: ${PIPELINE_ID})"
+
+while true; do
+    STATUS=$(curl --silent \
+        --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \
+        "${CI_API_V4_URL}/projects/${E2E_PROJECT_ENCODED}/pipelines/${PIPELINE_ID}" \
+        | jq -r '.status // empty')
+
+    echo -n "E2E pipeline ${PIPELINE_ID} status: ${STATUS}, "
+
+    case "$STATUS" in
+        success)
+            echo "E2E tests passed"
+            exit 0
+            ;;
+        failed)
+            echo "E2E tests failed"
+            exit 1
+            ;;
+        canceled|canceling)
+            echo "E2E tests canceled"
+            exit 1
+            ;;
+        skipped)
+            echo "E2E tests skipped"
+            exit 0
+            ;;
+        running|pending|created|waiting_for_resource|preparing)
+            echo "still running, checking again in 2 minutes..."
+            ;;
+        *)
+            echo "unknown status, checking again in 2 minutes..."
+            ;;
+    esac
+
+    sleep 120
+done

.gitlab/scripts/publish_layers.sh

@@ -167,3 +167,15 @@ while [ $latest_version -lt $VERSION ]; do
 done
 
 printf "[$REGION] Finished publishing layers...\n"
+
+if [ -n "${DOTENV:-}" ]; then
+    printf "[$REGION] Exporting layer ARN to %s...\n" "$DOTENV"
+    dotenv_key="${DOTENV_VAR_NAME:-EXTENSION_LAYER_ARN}"
+    latest_arn=$(aws lambda get-layer-version \
+        --layer-name "$LAYER_NAME" \
+        --version-number "$VERSION" \
+        --region "$REGION" \
+        --query 'LayerVersionArn' --output text)
+    echo "${dotenv_key}=${latest_arn}" >>"$DOTENV"
+    cat "$DOTENV"
+fi

.gitlab/templates/pipeline.yaml.tpl

@@ -1,9 +1,12 @@
+{{- $e2e_region := "us-west-2" -}}
+
 stages:
   - test
   - compile
   - build
   - integration-tests
   - self-monitoring
+  - e2e
   - sign
   - publish
 
@@ -141,6 +144,7 @@ sign layer ({{ $flavor.name }}):
     - .gitlab/scripts/sign_layers.sh prod
 
 {{ range $environment_name, $environment := (ds "environments").environments }}
+{{- $dotenv := printf "%s_%s.env" $flavor.suffix $environment_name }}
 
 publish layer {{ $environment_name }} ({{ $flavor.name }}):
   stage: publish
@@ -172,12 +176,21 @@ publish layer {{ $environment_name }} ({{ $flavor.name }}):
       - REGION: {{ range (ds "regions").regions }}
           - {{ .code }}
         {{- end}}
+{{- if eq $environment_name "sandbox" }}
+  artifacts:
+    reports:
+      dotenv: {{ $dotenv }}
+{{- end }}
   variables:
     LAYER_NAME_BASE_SUFFIX: {{ $flavor.layer_name_base_suffix }}
     ARCHITECTURE: {{ $flavor.arch }}
     LAYER_FILE: datadog_extension-{{ $flavor.suffix }}.zip
     ADD_LAYER_VERSION_PERMISSIONS: {{ $environment.add_layer_version_permissions }}
     AUTOMATICALLY_BUMP_VERSION: {{ $environment.automatically_bump_version }}
+{{- if eq $environment_name "sandbox" }}
+    LAYER_DESCRIPTION: ${CI_COMMIT_SHORT_SHA}
+    DOTENV: {{ $dotenv }}
+{{- end }}
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
   script:
@@ -217,6 +230,67 @@ publish layer [self-monitoring] ({{ $flavor.name }}):
 
 {{ end }}  # end flavors
 
+{{ range $f := (ds "flavors").flavors }}
+{{ if $f.needs_layer_publish }}
+{{- $dotenvE2E := printf "%s_sandbox_e2e.env" $f.suffix }}
+{{ with $environment := (ds "environments").environments.sandbox }}
+
+publish layer e2e sandbox ({{ $f.name }}):
+  stage: e2e
+  tags: ["arch:amd64"]
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+      when: on_success
+      variables:
+        LAYER_DESCRIPTION: $CI_COMMIT_TAG
+    - when: on_success
+      variables:
+        LAYER_DESCRIPTION: $CI_COMMIT_SHORT_SHA
+  needs:
+    - layer ({{ $f.name }})
+{{ if and (index $f "max_layer_compressed_size_mb") (index $f "max_layer_uncompressed_size_mb") }}
+    - check layer size ({{ $f.name }})
+{{ end }}
+  dependencies:
+    - layer ({{ $f.name }})
+  artifacts:
+    reports:
+      dotenv: {{ $dotenvE2E }}
+  variables:
+    LAYER_NAME_BASE_SUFFIX: {{ $f.layer_name_base_suffix }}
+    ARCHITECTURE: {{ $f.arch }}
+    LAYER_FILE: datadog_extension-{{ $f.suffix }}.zip
+    REGION: {{ $e2e_region }}
+    ADD_LAYER_VERSION_PERMISSIONS: {{ $environment.add_layer_version_permissions }}
+    AUTOMATICALLY_BUMP_VERSION: {{ $environment.automatically_bump_version }}
+    DOTENV: {{ $dotenvE2E }}
+  before_script:
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
+  script:
+    - .gitlab/scripts/publish_layers.sh
+
+e2e-suite ({{ $f.name }}):
+  stage: e2e
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  tags: ["arch:amd64"]
+  timeout: 3h
+  rules:
+    # Do not block on release, in case E2E is flaky
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+      when: on_success
+      allow_failure: true
+    - when: on_success
+  needs:
+    - job: "publish layer e2e sandbox ({{ $f.name }})"
+      artifacts: true
+  script:
+    - .gitlab/scripts/poll_e2e.sh
+
+{{ end }}
+{{ end }}
+{{ end }}
+
 {{ range $multi_arch_image_flavor := (ds "flavors").multi_arch_image_flavors }}
 
 publish private images ({{ $multi_arch_image_flavor.name }}):

bottlecap/Cargo.lock

@@ -40,6 +40,8 @@ sha2 = { version = "0.10", default-features = false }
 hex = { version = "0.4", default-features = false, features = ["std"] }
 base64 = { version = "0.22", default-features = false }
 rustls = { version = "0.23.18", default-features = false, features = ["aws-lc-rs"] }
+# Transitive via rustls. Pinned to >=0.103.13 so cargo audit passes (RUSTSEC-2026-0098, RUSTSEC-2026-0099, RUSTSEC-2026-0104).
+rustls-webpki = { version = "0.103.13", default-features = false }
 rustls-pemfile = { version = "2.0", default-features = false, features = ["std"] }
 rustls-pki-types = { version = "1.0", default-features = false }
 hyper-rustls = { version = "0.27.7", default-features = false }