Add telemetry properties for environment resolution and package installation (#25927)

eleanorjboyd · Copilot · web-flow · commit a8a4f70e2453 · 2026-04-23T17:23:26.000Z
Co-authored-by: Copilot &lt;copilot@github.com&gt;

1. Add `duration` to `INVOKE_TOOL` 
2. Add `resolveOutcome` to `configure_python_environment` 
3. Add `envType` to all tools that resolve an environment 
4. Add `packageCount` and `installerType` to `install_python_packages`
5. Add `responsePackageCount` to `get_python_environment_details`

---------

Co-authored-by: Copilot &lt;copilot@github.com&gt;
diff --git a/src/client/chat/baseTool.ts b/src/client/chat/baseTool.ts
@@ -16,8 +16,10 @@ import { IResourceReference, isCancellationError, resolveFilePath } from './util
 import { ErrorWithTelemetrySafeReason } from '../common/errors/errorUtils';
 import { sendTelemetryEvent } from '../telemetry';
 import { EventName } from '../telemetry/constants';
+import { StopWatch } from '../common/utils/stopWatch';
 
 export abstract class BaseTool<T extends IResourceReference> implements LanguageModelTool<T> {
+    protected extraTelemetryProperties: Record<string, string> = {};
     constructor(private readonly toolName: string) {}
 
     async invoke(
@@ -29,8 +31,10 @@ export abstract class BaseTool<T extends IResourceReference> implements Language
                 new LanguageModelTextPart('Cannot use this tool in an untrusted workspace.'),
             ]);
         }
+        this.extraTelemetryProperties = {};
         let error: Error | undefined;
         const resource = resolveFilePath(options.input.resourcePath);
+        const stopWatch = new StopWatch();
         try {
             return await this.invokeImpl(options, resource, token);
         } catch (ex) {
@@ -46,10 +50,11 @@ export abstract class BaseTool<T extends IResourceReference> implements Language
                     ? error.telemetrySafeReason
                     : 'error'
                 : undefined;
-            sendTelemetryEvent(EventName.INVOKE_TOOL, undefined, {
+            sendTelemetryEvent(EventName.INVOKE_TOOL, stopWatch.elapsedTime, {
                 toolName: this.toolName,
                 failed,
                 failureCategory,
+                ...this.extraTelemetryProperties,
             });
         }
     }
diff --git a/src/client/chat/configurePythonEnvTool.ts b/src/client/chat/configurePythonEnvTool.ts
@@ -18,6 +18,7 @@ import { ICodeExecutionService } from '../terminals/types';
 import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/terminalCodeExecution';
 import {
     getEnvDetailsForResponse,
+    getEnvTypeForTelemetry,
     getToolResponseIfNotebook,
     IResourceReference,
     isCancellationError,
@@ -58,6 +59,7 @@ export class ConfigurePythonEnvTool extends BaseTool<IResourceReference>
     ): Promise<LanguageModelToolResult> {
         const notebookResponse = getToolResponseIfNotebook(resource);
         if (notebookResponse) {
+            this.extraTelemetryProperties.resolveOutcome = 'notebook';
             return notebookResponse;
         }
 
@@ -67,6 +69,8 @@ export class ConfigurePythonEnvTool extends BaseTool<IResourceReference>
         );
 
         if (workspaceSpecificEnv) {
+            this.extraTelemetryProperties.resolveOutcome = 'existingWorkspaceEnv';
+            this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(workspaceSpecificEnv);
             return getEnvDetailsForResponse(
                 workspaceSpecificEnv,
                 this.api,
@@ -79,14 +83,17 @@ export class ConfigurePythonEnvTool extends BaseTool<IResourceReference>
 
         if (await this.createEnvTool.shouldCreateNewVirtualEnv(resource, token)) {
             try {
-                return await lm.invokeTool(CreateVirtualEnvTool.toolName, options, token);
+                const result = await lm.invokeTool(CreateVirtualEnvTool.toolName, options, token);
+                this.extraTelemetryProperties.resolveOutcome = 'createdVirtualEnv';
+                return result;
             } catch (ex) {
                 if (isCancellationError(ex)) {
                     const input: ISelectPythonEnvToolArguments = {
                         ...options.input,
                         reason: 'cancelled',
                     };
                     // If the user cancelled the tool, then we should invoke the select env tool.
+                    this.extraTelemetryProperties.resolveOutcome = 'selectedEnvAfterCancelledCreate';
                     return lm.invokeTool(SelectPythonEnvTool.toolName, { ...options, input }, token);
                 }
                 throw ex;
@@ -95,6 +102,7 @@ export class ConfigurePythonEnvTool extends BaseTool<IResourceReference>
             const input: ISelectPythonEnvToolArguments = {
                 ...options.input,
             };
+            this.extraTelemetryProperties.resolveOutcome = 'selectedEnv';
             return lm.invokeTool(SelectPythonEnvTool.toolName, { ...options, input }, token);
         }
     }
diff --git a/src/client/chat/getExecutableTool.ts b/src/client/chat/getExecutableTool.ts
@@ -19,6 +19,7 @@ import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/termin
 import {
     getEnvDisplayName,
     getEnvironmentDetails,
+    getEnvTypeForTelemetry,
     getToolResponseIfNotebook,
     IResourceReference,
     raceCancellationError,
@@ -53,6 +54,12 @@ export class GetExecutableTool extends BaseTool<IResourceReference> implements L
             return notebookResponse;
         }
 
+        const envPath = this.api.getActiveEnvironmentPath(resourcePath);
+        const environment = await raceCancellationError(this.api.resolveEnvironment(envPath), token);
+        if (environment) {
+            this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment);
+        }
+
         const message = await getEnvironmentDetails(
             resourcePath,
             this.api,
diff --git a/src/client/chat/getPythonEnvTool.ts b/src/client/chat/getPythonEnvTool.ts
@@ -17,7 +17,13 @@ import { IServiceContainer } from '../ioc/types';
 import { ICodeExecutionService } from '../terminals/types';
 import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/terminalCodeExecution';
 import { IProcessServiceFactory, IPythonExecutionFactory } from '../common/process/types';
-import { getEnvironmentDetails, getToolResponseIfNotebook, IResourceReference, raceCancellationError } from './utils';
+import {
+    getEnvironmentDetails,
+    getEnvTypeForTelemetry,
+    getToolResponseIfNotebook,
+    IResourceReference,
+    raceCancellationError,
+} from './utils';
 import { getPythonPackagesResponse } from './listPackagesTool';
 import { ITerminalHelper } from '../common/terminal/types';
 import { getEnvExtApi, useEnvExtension } from '../envExt/api.internal';
@@ -64,13 +70,16 @@ export class GetEnvironmentInfoTool extends BaseTool<IResourceReference>
                 'noEnvFound',
             );
         }
+        this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment);
 
         let packages = '';
+        let responsePackageCount = 0;
         if (useEnvExtension()) {
             const api = await getEnvExtApi();
             const env = await api.getEnvironment(resourcePath);
             const pkgs = env ? await api.getPackages(env) : [];
             if (pkgs && pkgs.length > 0) {
+                responsePackageCount = pkgs.length;
                 // Installed Python packages, each in the format <name> or <name> (<version>). The version may be omitted if unknown. Returns an empty array if no packages are installed.
                 const response = [
                     'Below is a list of the Python packages, each in the format <name> or <name> (<version>). The version may be omitted if unknown: ',
@@ -90,7 +99,10 @@ export class GetEnvironmentInfoTool extends BaseTool<IResourceReference>
                 resourcePath,
                 token,
             );
+            // Count lines starting with '- ' to get the number of packages
+            responsePackageCount = (packages.match(/^- /gm) || []).length;
         }
+        this.extraTelemetryProperties.responsePackageCount = String(responsePackageCount);
         const message = await getEnvironmentDetails(
             resourcePath,
             this.api,
diff --git a/src/client/chat/installPackagesTool.ts b/src/client/chat/installPackagesTool.ts
@@ -16,6 +16,7 @@ import { PythonExtension } from '../api/types';
 import { IServiceContainer } from '../ioc/types';
 import {
     getEnvDisplayName,
+    getEnvTypeForTelemetry,
     getToolResponseIfNotebook,
     IResourceReference,
     isCancellationError,
@@ -51,6 +52,7 @@ export class InstallPackagesTool extends BaseTool<IInstallPackageArgs>
     ): Promise<LanguageModelToolResult> {
         const packageCount = options.input.packageList.length;
         const packagePlurality = packageCount === 1 ? 'package' : 'packages';
+        this.extraTelemetryProperties.packageCount = String(packageCount);
         const notebookResponse = getToolResponseIfNotebook(resourcePath);
         if (notebookResponse) {
             return notebookResponse;
@@ -84,9 +86,11 @@ export class InstallPackagesTool extends BaseTool<IInstallPackageArgs>
                     'noEnvFound',
                 );
             }
+            this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment);
             const isConda = isCondaEnv(environment);
             const installers = this.serviceContainer.getAll<IModuleInstaller>(IModuleInstaller);
             const installerType = isConda ? ModuleInstallerType.Conda : ModuleInstallerType.Pip;
+            this.extraTelemetryProperties.installerType = isConda ? 'conda' : 'pip';
             const installer = installers.find((i) => i.type === installerType);
             if (!installer) {
                 throw new ErrorWithTelemetrySafeReason(
diff --git a/src/client/chat/utils.ts b/src/client/chat/utils.ts
@@ -76,6 +76,10 @@ export function isCondaEnv(env: ResolvedEnvironment) {
     return (env.environment?.type || '').toLowerCase() === 'conda';
 }
 
+export function getEnvTypeForTelemetry(env: ResolvedEnvironment): string {
+    return (env.environment?.type || 'unknown').toLowerCase();
+}
+
 export async function getEnvironmentDetails(
     resourcePath: Uri | undefined,
     api: PythonExtension['environments'],
diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts
@@ -1992,7 +1992,12 @@ export interface IEventNamePropertyMapping {
            "duration" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "owner": "donjayamanne" },
            "toolName" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "donjayamanne" },
             "failed": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Whether there was a failure. Common to most of the events.", "owner": "donjayamanne" },
-            "failureCategory": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"A reason that we generate (e.g. kerneldied, noipykernel, etc), more like a category of the error. Common to most of the events.", "owner": "donjayamanne" }
+            "failureCategory": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"A reason that we generate (e.g. kerneldied, noipykernel, etc), more like a category of the error. Common to most of the events.", "owner": "donjayamanne" },
+            "resolveOutcome": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Which code path resolved the environment in configure_python_environment.", "owner": "donjayamanne" },
+            "envType": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"The type of Python environment (e.g. venv, conda, system).", "owner": "donjayamanne" },
+            "packageCount": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Number of packages requested for installation (install_python_packages only).", "owner": "donjayamanne" },
+            "installerType": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Which installer was used: pip or conda (install_python_packages only).", "owner": "donjayamanne" },
+            "responsePackageCount": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Number of packages in the environment response (get_python_environment_details only).", "owner": "donjayamanne" }
        }
      */
     [EventName.INVOKE_TOOL]: {
@@ -2009,6 +2014,26 @@ export interface IEventNamePropertyMapping {
          * A reason the error was thrown.
          */
         failureCategory?: string;
+        /**
+         * Which code path resolved the environment (configure_python_environment only).
+         */
+        resolveOutcome?: string;
+        /**
+         * The type of Python environment (e.g. venv, conda, system).
+         */
+        envType?: string;
+        /**
+         * Number of packages requested for installation (install_python_packages only).
+         */
+        packageCount?: string;
+        /**
+         * Which installer was used: pip or conda (install_python_packages only).
+         */
+        installerType?: string;
+        /**
+         * Number of packages in the environment response (get_python_environment_details only).
+         */
+        responsePackageCount?: string;
     };
     /**
      * Telemetry event sent if and when user configure tests command. This command can be trigerred from multiple places in the extension. (Command palette, prompt etc.)
