diff --git a/bin/decrypt b/bin/decrypt index 8ca16f1..3773cf7 100755 --- a/bin/decrypt +++ b/bin/decrypt @@ -52,10 +52,10 @@ for YAML_PATH in ${YAML_PATHS[@]}; do # Start printing out the helpful debugging messages echo " -> Found ${#ENCRYPTED_VARIABLES[@]} encrypted variables, ${#ENCRYPTED_FILES[@]} files, and ${#ENCRYPTED_ADHOC_VARIABLES[@]} adhoc variables" - + cat <<-EOD - + When running with appropriate keys setup, the cryptic plugin will export the following environment variables: @@ -77,7 +77,7 @@ EOD # Spacer for visual blocking echo echo - echo "And the following files:" + echo "And the following files:" # Decrypt actual files for FILE_PATH in "${ENCRYPTED_FILES[@]}"; do @@ -96,4 +96,4 @@ EOD echo " -> ${FILE_PATH} ($(filesize "${REPO_ROOT}/${FILE_PATH}")kb${SKIPPED_MSG})" done fi -done \ No newline at end of file +done diff --git a/lib/common.sh b/lib/common.sh index 0874238..d7e265e 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -451,6 +451,9 @@ function find_repo_key() { # If we don't have a decrypted repo key, let's try decrypting one using an agent private key, if we have it if [[ -v "AGENT_PRIVATE_KEY_PATH" ]]; then + if [[ ! -f "${AGENT_PRIVATE_KEY_PATH}" ]]; then + die "Specified agent private key at '${AGENT_PRIVATE_KEY_PATH}' does not exist!" + fi RSA_FINGERPRINT=$(rsa_fingerprint "${AGENT_PRIVATE_KEY_PATH}") ENCRYPTED_REPO_KEY_PATH="${REPO_ROOT}/.buildkite/cryptic_repo_keys/repo_key.${RSA_FINGERPRINT:0:8}" if [[ -f "${ENCRYPTED_REPO_KEY_PATH}" ]]; then @@ -476,7 +479,7 @@ function find_yaml_paths() { vecho "Searching for '.yml' files with the pattern '${YAML_SEARCH_PATH}'" readarray -d '' YAML_PATHS < <(collect_glob_pattern "${YAML_SEARCH_PATH}") - + if [[ "${#YAML_PATHS[@]}" -lt 1 ]]; then die "Unable to find any .yml files in the given pattern '${YAML_SEARCH_PATH}'!" fi diff --git a/lib/yaml_extraction_prologue.sh b/lib/yaml_extraction_prologue.sh index 1168e73..9203090 100644 --- a/lib/yaml_extraction_prologue.sh +++ b/lib/yaml_extraction_prologue.sh @@ -8,75 +8,113 @@ fi # Extract the `variables:` section of a cryptic `pipeline.yml` plugin section function extract_encrypted_variables() { - # Iterate over the steps in the yaml file - (shyaml -q get-values-0 steps <"${1}" || true) | - while IFS='' read -r -d '' STEP; do - # For each step, get its list of plugins - (shyaml -q get-values-0 plugins <<<"${STEP}" || true) | - while IFS='' read -r -d '' PLUGINS; do - # Get the plugin names - (shyaml -q keys-0 <<<"${PLUGINS}" || true) | - while IFS='' read -r -d '' PLUGIN_NAME; do - # Skip plugins that are not named `cryptic` - if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then - continue - fi - # For each plugin, if its `cryptic`, extract the variables - (shyaml -q get-values-0 "${PLUGIN_NAME}.variables" <<<"${PLUGINS}" || true) | - while IFS='' read -r -d '' VAR; do - printf "%s\n" "${VAR}" + # Function to process steps recursively + function process_steps() { + local steps_input="$1" + # Iterate over the steps in the yaml file + (shyaml -q get-values-0 steps <<<"${steps_input}" || true) | + while IFS='' read -r -d '' STEP; do + # Check if this step has nested steps (is a group) + if (shyaml -q get-value steps <<<"${STEP}" >/dev/null 2>&1); then + # Recursively process nested steps + process_steps "${STEP}" + else + # For each step, get its list of plugins + (shyaml -q get-values-0 plugins <<<"${STEP}" || true) | + while IFS='' read -r -d '' PLUGINS; do + # Get the plugin names + (shyaml -q keys-0 <<<"${PLUGINS}" || true) | + while IFS='' read -r -d '' PLUGIN_NAME; do + # Skip plugins that are not named `cryptic` + if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then + continue + fi + # For each plugin, if its `cryptic`, extract the variables + (shyaml -q get-values-0 "${PLUGIN_NAME}.variables" <<<"${PLUGINS}" || true) | + while IFS='' read -r -d '' VAR; do + printf "%s\n" "${VAR}" + done + done done - done + fi done - done + } + + # Start processing from the root of the YAML file + process_steps "$(cat "${1}")" } # Extract all variables that match "CRYPTIC_ADHOC_SECRET_*" function extract_adhoc_encrypted_variables() { - # Iterate over any global env mappings - (shyaml -q keys-0 env <"${1}" || true) | - while IFS='' read -r -d '' VARNAME; do - if [[ "${VARNAME}" == CRYPTIC_ADHOC_SECRET_* ]]; then - printf "%s\n" "${VARNAME:21}=$(shyaml -q get-value env.${VARNAME} <"${1}")" - fi - done - - # Iterate over the steps in the yaml file - (shyaml -q get-values-0 steps <"${1}" || true) | - while IFS='' read -r -d '' STEP; do - (shyaml -q keys-0 env <<<"${STEP}" || true) | + function process_env_vars() { + local yaml_section="$1" + (shyaml -q keys-0 env <<<"${yaml_section}" || true) | while IFS='' read -r -d '' VARNAME; do if [[ "${VARNAME}" == CRYPTIC_ADHOC_SECRET_* ]]; then - printf "%s\n" "${VARNAME:21}=$(shyaml -q get-value env.${VARNAME} <<<"${STEP}")" + printf "%s\n" "${VARNAME:21}=$(shyaml -q get-value env.${VARNAME} <<<"${yaml_section}")" fi done - done + } + + function process_steps() { + local steps_input="$1" + # Iterate over the steps in the yaml file + (shyaml -q get-values-0 steps <<<"${steps_input}" || true) | + while IFS='' read -r -d '' STEP; do + # Check if this step has nested steps (is a group) + if (shyaml -q get-value steps <<<"${STEP}" >/dev/null 2>&1); then + # Recursively process nested steps + process_steps "${STEP}" + else + # Process environment variables for this step + process_env_vars "${STEP}" + fi + done + } + + # First process any global env mappings + process_env_vars "$(cat "${1}")" + + # Then process all steps recursively + process_steps "$(cat "${1}")" } # Extract the `files:` section of a cryptic `pipeline.yml` plugin section function extract_encrypted_files() { - # Iterate over the steps in the yaml file - (shyaml -q get-values-0 steps <"${1}" || true) | - while IFS='' read -r -d '' STEP; do - # For each step, get its list of plugins - (shyaml -q get-values-0 plugins <<<"${STEP}" || true) | - while IFS='' read -r -d '' PLUGINS; do - # Get the plugin names - (shyaml -q keys-0 <<<"${PLUGINS}" || true) | - while IFS='' read -r -d '' PLUGIN_NAME; do - # Skip plugins that are not named `cryptic` - if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then - continue - fi - # For each plugin, if its `cryptic`, extract the files - (shyaml -q get-values-0 "${PLUGIN_NAME}.files" <<<"${PLUGINS}" || true) | - while IFS='' read -r -d '' FILE; do - FILE="$(echo ${FILE} | tr -d '"')" - printf "%s\n" "${FILE}" + function process_steps() { + local steps_input="$1" + # Iterate over the steps in the yaml file + (shyaml -q get-values-0 steps <<<"${steps_input}" || true) | + while IFS='' read -r -d '' STEP; do + # Check if this step has nested steps (is a group) + if (shyaml -q get-value steps <<<"${STEP}" >/dev/null 2>&1); then + # Recursively process nested steps + process_steps "${STEP}" + else + # For each step, get its list of plugins + (shyaml -q get-values-0 plugins <<<"${STEP}" || true) | + while IFS='' read -r -d '' PLUGINS; do + # Get the plugin names + (shyaml -q keys-0 <<<"${PLUGINS}" || true) | + while IFS='' read -r -d '' PLUGIN_NAME; do + # Skip plugins that are not named `cryptic` + if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then + continue + fi + # For each plugin, if its `cryptic`, extract the files + (shyaml -q get-values-0 "${PLUGIN_NAME}.files" <<<"${PLUGINS}" || true) | + while IFS='' read -r -d '' FILE; do + FILE="$(echo ${FILE} | tr -d '"')" + printf "%s\n" "${FILE}" + done + done done - done + fi done - done + } + + # Start processing from the root of the YAML file + process_steps "$(cat "${1}")" } # Calculate the treehashes of each signed pipeline defined within a launching `.yml` file, @@ -87,76 +125,84 @@ function extract_pipeline_treehashes() { vecho "Extracting treehashes from '${YAML_PATH}'" - # Iterate over the steps in the yaml file - (shyaml -q get-values-0 steps <"${1}" || true) | - while IFS='' read -r -d '' STEP; do - # If this step is a `group` step, let's iterate over each of its steps - if shyaml -q get-value 'group' >/dev/null <<<"${STEP}"; then - (shyaml -q get-values-0 steps <<<"${STEP}" || true) | - while IFS='' read -r -d '' INNER_STEP; do - extract_plugin_treehashes "${INNER_STEP}" - done - else - extract_plugin_treehashes "${STEP}" - fi - done - - # Don't stay in `${REPO_ROOT}` - popd >/dev/null -} - -function extract_plugin_treehashes() { - # Get the list of plugins - (shyaml -q get-values-0 plugins <<<"${1}" || true) | - while IFS='' read -r -d '' PLUGINS; do - # Get the plugin names - (shyaml -q keys-0 <<<"${PLUGINS}" || true) | - while IFS='' read -r -d '' PLUGIN_NAME; do - # Skip plugins that are not named `cryptic` - if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then - continue - fi - - # For each plugin, if its `cryptic`, walk over the pipelines - (shyaml -q get-values-0 "${PLUGIN_NAME}.signed_pipelines" <<<"${PLUGINS}" || true) | - while IFS='' read -r -d '' PIPELINE; do - # For each signed pipeline, get its pipeline path and its inputs - PIPELINE_PATH="$(shyaml -q get-value "pipeline" <<<"${PIPELINE}" || true)" - - vecho " -> Found pipeline launch:" - vecho " -> ${PIPELINE_PATH}" - - # Start by calculating the treehash of the yaml file - INPUT_TREEHASHES=( "$(calc_treehash <<<"${PIPELINE_PATH}")" ) - - # Next, calculate the treehash of the rest of the glob patterns - readarray -d '' PATTERNS -t < <(shyaml -q get-values-0 "inputs" <<<"${PIPELINE}") - for PATTERN in "${PATTERNS[@]}"; do - HASH="$(collect_glob_pattern "${PATTERN}" | calc_treehash)" - vecho " + ${HASH} <- ${PATTERN}" - INPUT_TREEHASHES+=( "${HASH}" ) - done + # Function to process plugins and extract treehashes + function process_plugins() { + local STEP="$1" + # Get the list of plugins + (shyaml -q get-values-0 plugins <<<"${STEP}" || true) | + while IFS='' read -r -d '' PLUGINS; do + # Get the plugin names + (shyaml -q keys-0 <<<"${PLUGINS}" || true) | + while IFS='' read -r -d '' PLUGIN_NAME; do + # Skip plugins that are not named `cryptic` + if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then + continue + fi - # Calculate full treehash - FULL_TREEHASH="$(printf "%s" "${INPUT_TREEHASHES[@]}" | calc_shasum)" - vecho " ∟ ${FULL_TREEHASH}" - - # If `signature_file` is defined, use it! - local BASE64_ENCRYPTED_TREEHASH="" - local TREEHASH_FILE_SOURCE="" - if shyaml get-value "signature_file" <<<"${PIPELINE}" >/dev/null; then - TREEHASH_FILE_SOURCE="$(shyaml -q get-value "signature_file" <<<"${PIPELINE}")" - if [[ -f "${TREEHASH_FILE_SOURCE}" ]]; then - BASE64_ENCRYPTED_TREEHASH="$(base64enc <"${TREEHASH_FILE_SOURCE}")" + # For each plugin, if its `cryptic`, walk over the pipelines + (shyaml -q get-values-0 "${PLUGIN_NAME}.signed_pipelines" <<<"${PLUGINS}" || true) | + while IFS='' read -r -d '' PIPELINE; do + # For each signed pipeline, get its pipeline path and its inputs + PIPELINE_PATH="$(shyaml -q get-value "pipeline" <<<"${PIPELINE}" || true)" + + vecho " -> Found pipeline launch:" + vecho " -> ${PIPELINE_PATH}" + + # Start by calculating the treehash of the yaml file + INPUT_TREEHASHES=( "$(calc_treehash <<<"${PIPELINE_PATH}")" ) + + # Next, calculate the treehash of the rest of the glob patterns + readarray -d '' PATTERNS -t < <(shyaml -q get-values-0 "inputs" <<<"${PIPELINE}") + for PATTERN in "${PATTERNS[@]}"; do + HASH="$(collect_glob_pattern "${PATTERN}" | calc_treehash)" + vecho " + ${HASH} <- ${PATTERN}" + INPUT_TREEHASHES+=( "${HASH}" ) + done + + # Calculate full treehash + FULL_TREEHASH="$(printf "%s" "${INPUT_TREEHASHES[@]}" | calc_shasum)" + vecho " ∟ ${FULL_TREEHASH}" + + # If `signature_file` is defined, use it! + local BASE64_ENCRYPTED_TREEHASH="" + local TREEHASH_FILE_SOURCE="" + if shyaml get-value "signature_file" <<<"${PIPELINE}" >/dev/null; then + TREEHASH_FILE_SOURCE="$(shyaml -q get-value "signature_file" <<<"${PIPELINE}")" + if [[ -f "${TREEHASH_FILE_SOURCE}" ]]; then + BASE64_ENCRYPTED_TREEHASH="$(base64enc <"${TREEHASH_FILE_SOURCE}")" + fi + else + # Try to extract the signature from the yaml directly too + BASE64_ENCRYPTED_TREEHASH="$(shyaml -q get-value "signature" <<<"${PIPELINE}" || true)" fi - else - # Try to extract the signature from the yaml directly too - BASE64_ENCRYPTED_TREEHASH="$(shyaml -q get-value "signature" <<<"${PIPELINE}" || true)" - fi - # Print out treehash and pipeline path - printf "%s&%s&%s&%s\n" "${PIPELINE_PATH}" "${FULL_TREEHASH}" "${BASE64_ENCRYPTED_TREEHASH}" "${TREEHASH_FILE_SOURCE}" + # Print out treehash and pipeline path + printf "%s&%s&%s&%s\n" "${PIPELINE_PATH}" "${FULL_TREEHASH}" "${BASE64_ENCRYPTED_TREEHASH}" "${TREEHASH_FILE_SOURCE}" + done done done - done + } + + # Function to process steps recursively + function process_steps() { + local steps_input="$1" + # Iterate over the steps in the yaml file + (shyaml -q get-values-0 steps <<<"${steps_input}" || true) | + while IFS='' read -r -d '' STEP; do + # Check if this step has nested steps (is a group) + if (shyaml -q get-value steps <<<"${STEP}" >/dev/null 2>&1); then + # Recursively process nested steps + process_steps "${STEP}" + else + # Process plugins for this step + process_plugins "${STEP}" + fi + done + } + + # Start processing from the root of the YAML file + process_steps "$(cat "${1}")" + + # Don't stay in `${REPO_ROOT}` + popd >/dev/null }