Skip to content

Add ability to use ddtrace_serverless wheels in layer build #777

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
emmettbutler merged 11 commits into from
Apr 2, 2026
Merged

Add ability to use ddtrace_serverless wheels in layer build #777

Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
429b26d
Revert "ci: revert PRs #762 and #775 (#776)"
emmettbutler Apr 2, 2026
a95762e
try a different way of checking success
emmettbutler Apr 2, 2026
aab590e
refactor to only do docker build once
emmettbutler Apr 2, 2026
3c64f8b
oh yeah
emmettbutler Apr 2, 2026
ae5c082
catch error code
emmettbutler Apr 2, 2026
803586b
bite the proverbial bullet
emmettbutler Apr 2, 2026
0c808bf
*shakes fist* ....bash!!!
emmettbutler Apr 2, 2026
2d7d300
more handling
emmettbutler Apr 2, 2026
e401b19
debug output
emmettbutler Apr 2, 2026
95a559e
construct basename when given a wheel directly
emmettbutler Apr 2, 2026
ccb70d8
handle dotslash in basename
emmettbutler Apr 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 29 additions & 12 deletions scripts/build_layers.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,22 +98,41 @@ function make_path_absolute {
echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
}

function search_wheel {
# Args: [wheel base name] [index]

WHEEL_BASENAME=$1
INDEX=$2

SEARCH_PATTERN="${WHEEL_BASENAME}-[^\"]*${PY_TAG}[^\"]*${PLATFORM}[^\"]*\.whl"
INDEX_URL="${S3_BASE}/index-${INDEX}.html"
echo "Searching for wheel ${SEARCH_PATTERN}"
export WHEEL_FILE=$(curl -sSfL ${INDEX_URL} | grep -o "$SEARCH_PATTERN" | head -n 1)
if [ ! -z "${WHEEL_FILE}" ]; then
curl -sSfL "${S3_BASE}/${WHEEL_FILE}" -o "${WHEEL_FILE}"
echo "Using S3 wheel: ${WHEEL_FILE}"
replace_ddtrace_dep "${WHEEL_BASENAME} = { file = \"${WHEEL_FILE}\" }"
fi
}

function docker_build_zip {
# Args: [python version] [zip destination]
# Args: [python version] [zip destination] [wheel base name] [index]

destination=$(make_path_absolute $2)
arch=$3
wheel_basename=$4
index=$5

# Restore pyproject.toml to a clean state for each build iteration
cp pyproject.toml.bak pyproject.toml

# Replace ddtrace source if necessary
if [ -n "$DD_TRACE_COMMIT" ]; then
replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"$DD_TRACE_COMMIT\" }"
replace_ddtrace_dep "${wheel_basename} = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"$DD_TRACE_COMMIT\" }"
elif [ -n "$DD_TRACE_COMMIT_BRANCH" ]; then
replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", branch = \"$DD_TRACE_COMMIT_BRANCH\" }"
replace_ddtrace_dep "${wheel_basename} = { git = \"https://github.com/DataDog/dd-trace-py.git\", branch = \"$DD_TRACE_COMMIT_BRANCH\" }"
elif [ -n "$DD_TRACE_WHEEL" ]; then
replace_ddtrace_dep "ddtrace = { file = \"$DD_TRACE_WHEEL\" }"
replace_ddtrace_dep "${wheel_basename} = { file = \"$DD_TRACE_WHEEL\" }"
elif [ -n "$UPSTREAM_PIPELINE_ID" ]; then
S3_BASE="https://dd-trace-py-builds.s3.amazonaws.com/${UPSTREAM_PIPELINE_ID}"
if [ "${arch}" = "amd64" ]; then
Expand All @@ -122,15 +141,9 @@ function docker_build_zip {
PLATFORM="manylinux2014_aarch64"
fi
PY_TAG="cp$(echo "$1" | tr -d '.')"
WHEEL_FILE=$(curl -sSfL "${S3_BASE}/index-manylinux2014.html" \
| grep -o "ddtrace-[^\"]*${PY_TAG}[^\"]*${PLATFORM}[^\"]*\.whl" \
| head -n 1)
search_wheel ${wheel_basename} ${index}
if [ -z "${WHEEL_FILE}" ]; then
echo "No S3 wheel found for ${PY_TAG} ${PLATFORM}, using default pyproject.toml version"
else
curl -sSfL "${S3_BASE}/${WHEEL_FILE}" -o "${WHEEL_FILE}"
echo "Using S3 wheel: ${WHEEL_FILE}"
replace_ddtrace_dep "ddtrace = { file = \"${WHEEL_FILE}\" }"
fi
fi

Expand All @@ -149,6 +162,7 @@ function docker_build_zip {

rm -rf $temp_dir
echo "Done creating archive $destination"
rm pyproject.toml.bak
}

rm -rf $LAYER_DIR
Expand All @@ -159,7 +173,10 @@ do
for architecture in "${ARCHS[@]}"
do
echo "Building layer for Python ${python_version} arch=${architecture}"
docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-${architecture}-${python_version}.zip ${architecture}
docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-${architecture}-${python_version}.zip ${architecture} "ddtrace_serverless" "serverless" || true
if [ -f pyproject.toml.bak ]; then # true means the previous attempt failed
docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-${architecture}-${python_version}.zip ${architecture} "ddtrace" "manylinux2014"
fi
done
done

Expand Down
Loading