Skip to content

Adapt secret handling to pipeline groups #32

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
maleadt wants to merge 3 commits into from
Closed

Adapt secret handling to pipeline groups #32

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fb8bd63
Gracefully handle nonexisting agent keys.
maleadt Nov 8, 2024
4b6be21
Update scripts to handle nested steps.
maleadt Nov 8, 2024
71a4ff8
Whitespace fixes.
maleadt Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions bin/decrypt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,10 @@ for YAML_PATH in ${YAML_PATHS[@]}; do

# Start printing out the helpful debugging messages
echo " -> Found ${#ENCRYPTED_VARIABLES[@]} encrypted variables, ${#ENCRYPTED_FILES[@]} files, and ${#ENCRYPTED_ADHOC_VARIABLES[@]} adhoc variables"


cat <<-EOD

When running with appropriate keys setup, the cryptic plugin
will export the following environment variables:

Expand All @@ -77,7 +77,7 @@ EOD
# Spacer for visual blocking
echo
echo
echo "And the following files:"
echo "And the following files:"

# Decrypt actual files
for FILE_PATH in "${ENCRYPTED_FILES[@]}"; do
Expand All @@ -96,4 +96,4 @@ EOD
echo " -> ${FILE_PATH} ($(filesize "${REPO_ROOT}/${FILE_PATH}")kb${SKIPPED_MSG})"
done
fi
done
done
5 changes: 4 additions & 1 deletion lib/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,9 @@ function find_repo_key() {

# If we don't have a decrypted repo key, let's try decrypting one using an agent private key, if we have it
if [[ -v "AGENT_PRIVATE_KEY_PATH" ]]; then
if [[ ! -f "${AGENT_PRIVATE_KEY_PATH}" ]]; then
die "Specified agent private key at '${AGENT_PRIVATE_KEY_PATH}' does not exist!"
fi
RSA_FINGERPRINT=$(rsa_fingerprint "${AGENT_PRIVATE_KEY_PATH}")
ENCRYPTED_REPO_KEY_PATH="${REPO_ROOT}/.buildkite/cryptic_repo_keys/repo_key.${RSA_FINGERPRINT:0:8}"
if [[ -f "${ENCRYPTED_REPO_KEY_PATH}" ]]; then
Expand All @@ -476,7 +479,7 @@ function find_yaml_paths() {

vecho "Searching for '.yml' files with the pattern '${YAML_SEARCH_PATH}'"
readarray -d '' YAML_PATHS < <(collect_glob_pattern "${YAML_SEARCH_PATH}")

if [[ "${#YAML_PATHS[@]}" -lt 1 ]]; then
die "Unable to find any .yml files in the given pattern '${YAML_SEARCH_PATH}'!"
fi
Expand Down
286 changes: 166 additions & 120 deletions lib/yaml_extraction_prologue.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,75 +8,113 @@ fi

# Extract the `variables:` section of a cryptic `pipeline.yml` plugin section
function extract_encrypted_variables() {
# Iterate over the steps in the yaml file
(shyaml -q get-values-0 steps <"${1}" || true) |
while IFS='' read -r -d '' STEP; do
# For each step, get its list of plugins
(shyaml -q get-values-0 plugins <<<"${STEP}" || true) |
while IFS='' read -r -d '' PLUGINS; do
# Get the plugin names
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PLUGIN_NAME; do
# Skip plugins that are not named `cryptic`
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
continue
fi
# For each plugin, if its `cryptic`, extract the variables
(shyaml -q get-values-0 "${PLUGIN_NAME}.variables" <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' VAR; do
printf "%s\n" "${VAR}"
# Function to process steps recursively
function process_steps() {
local steps_input="$1"
# Iterate over the steps in the yaml file
(shyaml -q get-values-0 steps <<<"${steps_input}" || true) |
while IFS='' read -r -d '' STEP; do
# Check if this step has nested steps (is a group)
if (shyaml -q get-value steps <<<"${STEP}" >/dev/null 2>&1); then
# Recursively process nested steps
process_steps "${STEP}"
else
# For each step, get its list of plugins
(shyaml -q get-values-0 plugins <<<"${STEP}" || true) |
while IFS='' read -r -d '' PLUGINS; do
# Get the plugin names
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PLUGIN_NAME; do
# Skip plugins that are not named `cryptic`
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
continue
fi
# For each plugin, if its `cryptic`, extract the variables
(shyaml -q get-values-0 "${PLUGIN_NAME}.variables" <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' VAR; do
printf "%s\n" "${VAR}"
done
done
done
done
fi
done
done
}

# Start processing from the root of the YAML file
process_steps "$(cat "${1}")"
}

# Extract all variables that match "CRYPTIC_ADHOC_SECRET_*"
function extract_adhoc_encrypted_variables() {
# Iterate over any global env mappings
(shyaml -q keys-0 env <"${1}" || true) |
while IFS='' read -r -d '' VARNAME; do
if [[ "${VARNAME}" == CRYPTIC_ADHOC_SECRET_* ]]; then
printf "%s\n" "${VARNAME:21}=$(shyaml -q get-value env.${VARNAME} <"${1}")"
fi
done

# Iterate over the steps in the yaml file
(shyaml -q get-values-0 steps <"${1}" || true) |
while IFS='' read -r -d '' STEP; do
(shyaml -q keys-0 env <<<"${STEP}" || true) |
function process_env_vars() {
local yaml_section="$1"
(shyaml -q keys-0 env <<<"${yaml_section}" || true) |
while IFS='' read -r -d '' VARNAME; do
if [[ "${VARNAME}" == CRYPTIC_ADHOC_SECRET_* ]]; then
printf "%s\n" "${VARNAME:21}=$(shyaml -q get-value env.${VARNAME} <<<"${STEP}")"
printf "%s\n" "${VARNAME:21}=$(shyaml -q get-value env.${VARNAME} <<<"${yaml_section}")"
fi
done
done
}

function process_steps() {
local steps_input="$1"
# Iterate over the steps in the yaml file
(shyaml -q get-values-0 steps <<<"${steps_input}" || true) |
while IFS='' read -r -d '' STEP; do
# Check if this step has nested steps (is a group)
if (shyaml -q get-value steps <<<"${STEP}" >/dev/null 2>&1); then
# Recursively process nested steps
process_steps "${STEP}"
else
# Process environment variables for this step
process_env_vars "${STEP}"
fi
done
}

# First process any global env mappings
process_env_vars "$(cat "${1}")"

# Then process all steps recursively
process_steps "$(cat "${1}")"
}

# Extract the `files:` section of a cryptic `pipeline.yml` plugin section
function extract_encrypted_files() {
# Iterate over the steps in the yaml file
(shyaml -q get-values-0 steps <"${1}" || true) |
while IFS='' read -r -d '' STEP; do
# For each step, get its list of plugins
(shyaml -q get-values-0 plugins <<<"${STEP}" || true) |
while IFS='' read -r -d '' PLUGINS; do
# Get the plugin names
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PLUGIN_NAME; do
# Skip plugins that are not named `cryptic`
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
continue
fi
# For each plugin, if its `cryptic`, extract the files
(shyaml -q get-values-0 "${PLUGIN_NAME}.files" <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' FILE; do
FILE="$(echo ${FILE} | tr -d '"')"
printf "%s\n" "${FILE}"
function process_steps() {
local steps_input="$1"
# Iterate over the steps in the yaml file
(shyaml -q get-values-0 steps <<<"${steps_input}" || true) |
while IFS='' read -r -d '' STEP; do
# Check if this step has nested steps (is a group)
if (shyaml -q get-value steps <<<"${STEP}" >/dev/null 2>&1); then
# Recursively process nested steps
process_steps "${STEP}"
else
# For each step, get its list of plugins
(shyaml -q get-values-0 plugins <<<"${STEP}" || true) |
while IFS='' read -r -d '' PLUGINS; do
# Get the plugin names
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PLUGIN_NAME; do
# Skip plugins that are not named `cryptic`
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
continue
fi
# For each plugin, if its `cryptic`, extract the files
(shyaml -q get-values-0 "${PLUGIN_NAME}.files" <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' FILE; do
FILE="$(echo ${FILE} | tr -d '"')"
printf "%s\n" "${FILE}"
done
done
done
done
fi
done
done
}

# Start processing from the root of the YAML file
process_steps "$(cat "${1}")"
}

# Calculate the treehashes of each signed pipeline defined within a launching `.yml` file,
Expand All @@ -87,76 +125,84 @@ function extract_pipeline_treehashes() {

vecho "Extracting treehashes from '${YAML_PATH}'"

# Iterate over the steps in the yaml file
(shyaml -q get-values-0 steps <"${1}" || true) |
while IFS='' read -r -d '' STEP; do
# If this step is a `group` step, let's iterate over each of its steps
if shyaml -q get-value 'group' >/dev/null <<<"${STEP}"; then
(shyaml -q get-values-0 steps <<<"${STEP}" || true) |
while IFS='' read -r -d '' INNER_STEP; do
extract_plugin_treehashes "${INNER_STEP}"
done
else
extract_plugin_treehashes "${STEP}"
fi
done

# Don't stay in `${REPO_ROOT}`
popd >/dev/null
}

function extract_plugin_treehashes() {
# Get the list of plugins
(shyaml -q get-values-0 plugins <<<"${1}" || true) |
while IFS='' read -r -d '' PLUGINS; do
# Get the plugin names
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PLUGIN_NAME; do
# Skip plugins that are not named `cryptic`
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
continue
fi

# For each plugin, if its `cryptic`, walk over the pipelines
(shyaml -q get-values-0 "${PLUGIN_NAME}.signed_pipelines" <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PIPELINE; do
# For each signed pipeline, get its pipeline path and its inputs
PIPELINE_PATH="$(shyaml -q get-value "pipeline" <<<"${PIPELINE}" || true)"

vecho " -> Found pipeline launch:"
vecho " -> ${PIPELINE_PATH}"

# Start by calculating the treehash of the yaml file
INPUT_TREEHASHES=( "$(calc_treehash <<<"${PIPELINE_PATH}")" )

# Next, calculate the treehash of the rest of the glob patterns
readarray -d '' PATTERNS -t < <(shyaml -q get-values-0 "inputs" <<<"${PIPELINE}")
for PATTERN in "${PATTERNS[@]}"; do
HASH="$(collect_glob_pattern "${PATTERN}" | calc_treehash)"
vecho " + ${HASH} <- ${PATTERN}"
INPUT_TREEHASHES+=( "${HASH}" )
done
# Function to process plugins and extract treehashes
function process_plugins() {
local STEP="$1"
# Get the list of plugins
(shyaml -q get-values-0 plugins <<<"${STEP}" || true) |
while IFS='' read -r -d '' PLUGINS; do
# Get the plugin names
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PLUGIN_NAME; do
# Skip plugins that are not named `cryptic`
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
continue
fi

# Calculate full treehash
FULL_TREEHASH="$(printf "%s" "${INPUT_TREEHASHES[@]}" | calc_shasum)"
vecho " ∟ ${FULL_TREEHASH}"

# If `signature_file` is defined, use it!
local BASE64_ENCRYPTED_TREEHASH=""
local TREEHASH_FILE_SOURCE=""
if shyaml get-value "signature_file" <<<"${PIPELINE}" >/dev/null; then
TREEHASH_FILE_SOURCE="$(shyaml -q get-value "signature_file" <<<"${PIPELINE}")"
if [[ -f "${TREEHASH_FILE_SOURCE}" ]]; then
BASE64_ENCRYPTED_TREEHASH="$(base64enc <"${TREEHASH_FILE_SOURCE}")"
# For each plugin, if its `cryptic`, walk over the pipelines
(shyaml -q get-values-0 "${PLUGIN_NAME}.signed_pipelines" <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PIPELINE; do
# For each signed pipeline, get its pipeline path and its inputs
PIPELINE_PATH="$(shyaml -q get-value "pipeline" <<<"${PIPELINE}" || true)"

vecho " -> Found pipeline launch:"
vecho " -> ${PIPELINE_PATH}"

# Start by calculating the treehash of the yaml file
INPUT_TREEHASHES=( "$(calc_treehash <<<"${PIPELINE_PATH}")" )

# Next, calculate the treehash of the rest of the glob patterns
readarray -d '' PATTERNS -t < <(shyaml -q get-values-0 "inputs" <<<"${PIPELINE}")
for PATTERN in "${PATTERNS[@]}"; do
HASH="$(collect_glob_pattern "${PATTERN}" | calc_treehash)"
vecho " + ${HASH} <- ${PATTERN}"
INPUT_TREEHASHES+=( "${HASH}" )
done

# Calculate full treehash
FULL_TREEHASH="$(printf "%s" "${INPUT_TREEHASHES[@]}" | calc_shasum)"
vecho " ∟ ${FULL_TREEHASH}"

# If `signature_file` is defined, use it!
local BASE64_ENCRYPTED_TREEHASH=""
local TREEHASH_FILE_SOURCE=""
if shyaml get-value "signature_file" <<<"${PIPELINE}" >/dev/null; then
TREEHASH_FILE_SOURCE="$(shyaml -q get-value "signature_file" <<<"${PIPELINE}")"
if [[ -f "${TREEHASH_FILE_SOURCE}" ]]; then
BASE64_ENCRYPTED_TREEHASH="$(base64enc <"${TREEHASH_FILE_SOURCE}")"
fi
else
# Try to extract the signature from the yaml directly too
BASE64_ENCRYPTED_TREEHASH="$(shyaml -q get-value "signature" <<<"${PIPELINE}" || true)"
fi
else
# Try to extract the signature from the yaml directly too
BASE64_ENCRYPTED_TREEHASH="$(shyaml -q get-value "signature" <<<"${PIPELINE}" || true)"
fi

# Print out treehash and pipeline path
printf "%s&%s&%s&%s\n" "${PIPELINE_PATH}" "${FULL_TREEHASH}" "${BASE64_ENCRYPTED_TREEHASH}" "${TREEHASH_FILE_SOURCE}"
# Print out treehash and pipeline path
printf "%s&%s&%s&%s\n" "${PIPELINE_PATH}" "${FULL_TREEHASH}" "${BASE64_ENCRYPTED_TREEHASH}" "${TREEHASH_FILE_SOURCE}"
done
done
done
done
}

# Function to process steps recursively
function process_steps() {
local steps_input="$1"
# Iterate over the steps in the yaml file
(shyaml -q get-values-0 steps <<<"${steps_input}" || true) |
while IFS='' read -r -d '' STEP; do
# Check if this step has nested steps (is a group)
if (shyaml -q get-value steps <<<"${STEP}" >/dev/null 2>&1); then
# Recursively process nested steps
process_steps "${STEP}"
else
# Process plugins for this step
process_plugins "${STEP}"
fi
done
}

# Start processing from the root of the YAML file
process_steps "$(cat "${1}")"

# Don't stay in `${REPO_ROOT}`
popd >/dev/null
}