Improve token handling and cleanup in docker.md

Refactor token loading and cleanup functions in docker script.

Fixes issue with this error during cleanup
    WRITE ERROR: VS30063: You are not authorized to access https://dev.azure.com.

Also fixed issue with hard code AZP_TOKEN_FILE location being a hard coded path /azp/.token which may not exists.  Changed it to use the same location as the script location.

Author: ianwallen

docs/pipelines/agents/docker.md

@@ -316,42 +316,58 @@ Next, create the Dockerfile.
     #!/bin/bash
     set -e
 
-    if [ -z "${AZP_URL}" ]; then
-      echo 1>&2 "error: missing AZP_URL environment variable"
-      exit 1
-    fi
+    # Load a token either from the environment variable or by using the service principal credentials.
+    load_azp_token() {
+      if [ -n "$AZP_CLIENTID" ]; then
+        if [ -z "$AZP_CLIENTSECRET" ]; then
+          echo 1>&2 "error: AZP_CLIENTSECRET must be set when AZP_CLIENTID is used"
+          exit 1
+        fi
+
+        if [ -z "$AZP_TENANTID" ]; then
+          echo 1>&2 "error: AZP_TENANTID must be set when AZP_CLIENTID is used"
+          exit 1
+        fi
+
+        echo "Using service principal credentials to get token"
+        az login --allow-no-subscriptions --service-principal --username "$AZP_CLIENTID" --password "$AZP_CLIENTSECRET" --tenant "$AZP_TENANTID"
+        # adapted from https://learn.microsoft.com/en-us/azure/databricks/dev-tools/user-aad-token
+        AZP_TOKEN=$(az account get-access-token --query accessToken --output tsv)
+        echo "Token retrieved"
+ 
+        # Ensure credentials are not visible to the agent by un-exporting the variable
+        export -n AZP_CLIENTSECRET
+      fi
 
-    if [ -n "$AZP_CLIENTID" ]; then
-      echo "Using service principal credentials to get token"
-      az login --allow-no-subscriptions --service-principal --username "$AZP_CLIENTID" --password "$AZP_CLIENTSECRET" --tenant "$AZP_TENANTID"
-      # adapted from https://learn.microsoft.com/en-us/azure/databricks/dev-tools/user-aad-token
-      AZP_TOKEN=$(az account get-access-token --query accessToken --output tsv)
-      echo "Token retrieved"
-    fi
+      if [ -z "${AZP_TOKEN_FILE}" ]; then
+        if [ -z "${AZP_TOKEN}" ]; then
+          echo 1>&2 "error: missing AZP_TOKEN environment variable"
+          exit 1
+        fi
 
-    if [ -z "${AZP_TOKEN_FILE}" ]; then
-      if [ -z "${AZP_TOKEN}" ]; then
-        echo 1>&2 "error: missing AZP_TOKEN environment variable"
-        exit 1
+        AZP_TOKEN_FILE="$(dirname "$0")/.token"
       fi
 
-      AZP_TOKEN_FILE="/azp/.token"
-      echo -n "${AZP_TOKEN}" > "${AZP_TOKEN_FILE}"
-    fi
-
-    unset AZP_CLIENTSECRET
-    unset AZP_TOKEN
+      if [ -n "${AZP_TOKEN}" ]; then
+           echo -n "${AZP_TOKEN}" > "${AZP_TOKEN_FILE}"
+      fi
 
-    if [ -n "${AZP_WORK}" ]; then
-      mkdir -p "${AZP_WORK}"
-    fi
+      # Ensure credentials are not visible to the agent by un-exporting the variable
+      export -n AZP_TOKEN
+    }
 
+    # Cleanup function to remove the agent configuration.
     cleanup() {
       trap "" EXIT
 
       if [ -e ./config.sh ]; then
         print_header "Cleanup. Removing Azure Pipelines agent..."
 
+        # Ensure we have a new token if using service principal credentials, as the old one might have expired between the time it was waiting to run a job and now.
+        if [ -n "$AZP_CLIENTID" ]; then
+          load_azp_token
+        fi
+    
         # If the agent has some running jobs, the configuration removal process will fail.
         # So, give it some time to finish the job.
         while true; do
@@ -369,8 +385,20 @@ Next, create the Dockerfile.
       echo -e "\n${lightcyan}$1${nocolor}\n"
     }
 
+    if [ -z "${AZP_URL}" ]; then
+      echo 1>&2 "error: missing AZP_URL environment variable"
+      exit 1
+    fi
+
+    # Load the AZP token for initial setup.
+    load_azp_token
+    
+    if [ -n "${AZP_WORK}" ]; then
+      mkdir -p "${AZP_WORK}"
+    fi
+    
     # Let the agent ignore the token env variables
-    export VSO_AGENT_IGNORE="AZP_TOKEN,AZP_TOKEN_FILE"
+    export VSO_AGENT_IGNORE="AZP_TOKEN,AZP_TOKEN_FILE,AZP_CLIENTSECRET"
 
     print_header "1. Determining matching Azure Pipelines agent..."