Add HTTP proxy and corporate-CA support for SDK and bundled CLI

packages/databricks-vscode/package.json

@@ -1856,6 +1856,14 @@
                         "type": "number",
                         "default": 5,
                         "description": "The interval in minutes at which the remote state of the bundle is refreshed."
+                    },
+                    "databricks.proxy.strictSSL": {
+                        "type": "boolean",
+                        "markdownDescription": "Controls whether TLS certificates are verified for the extension's Databricks SDK calls. When unset, the built-in `#http.proxyStrictSSL#` value is used. Set to `false` only when a corporate CA certificate cannot be resolved from the system trust store."
+                    },
+                    "databricks.proxy.caCert": {
+                        "type": "string",
+                        "markdownDescription": "Absolute path to a PEM bundle of additional CA certificates to trust for the extension's Databricks SDK calls, merged with the OS trust store and Node's built-in roots. Use when a corporate CA cannot be read from the system store."
                     }
                 }
             }
@@ -1875,7 +1883,7 @@
         "version": "1.15.0"
     },
     "scripts": {
-        "vscode:prepublish": "rm -rf out && yarn run package:compile && yarn run package:wrappers:write && yarn run package:jupyter-init-script:write && yarn run package:copy-webview-toolkit && yarn run package:copy-markdown-it && yarn run package:copy-highlight && yarn run generate-telemetry",
+        "vscode:prepublish": "rm -rf out && yarn run package:compile && yarn run package:wrappers:write && yarn run package:jupyter-init-script:write && yarn run package:copy-webview-toolkit && yarn run package:copy-markdown-it && yarn run package:copy-highlight && yarn run package:copy-windows-ca-certs && yarn run generate-telemetry",
         "package": "vsce package --baseContentUrl https://github.com/databricks/databricks-vscode/blob/${TAG:-main}/packages/databricks-vscode --baseImagesUrl https://raw.githubusercontent.com/databricks/databricks-vscode/${TAG:-main}/packages/databricks-vscode",
         "package:linux:x64": "./scripts/package-vsix.sh linux-x64",
         "package:linux:arm64": "./scripts/package-vsix.sh linux-arm64",
@@ -1893,7 +1901,8 @@
         "package:copy-webview-toolkit": "cp ./node_modules/@vscode/webview-ui-toolkit/dist/toolkit.js ./out/toolkit.js",
         "package:copy-markdown-it": "cp ./node_modules/markdown-it/dist/browser/markdown-it.umd.min.js ./out/markdown-it.min.js",
         "package:copy-highlight": "esbuild ./node_modules/highlight.js/lib/common.js --bundle --format=iife --global-name=hljs --outfile=out/highlight.min.js --minify",
-        "esbuild:base": "esbuild ./src/extension.ts --bundle --outfile=out/extension.js --external:vscode --format=cjs --platform=node --sourcemap --target=es2019",
+        "package:copy-windows-ca-certs": "bash ./scripts/copy-windows-ca-certs.sh",
+        "esbuild:base": "esbuild ./src/extension.ts --bundle --outfile=out/extension.js --external:vscode --external:@vscode/windows-ca-certs --format=cjs --platform=node --sourcemap --target=es2019",
         "build": "yarn run package:wrappers:write && yarn run package:jupyter-init-script:write && tsc --build --force",
         "watch": "yarn run package:wrappers:write && yarn run package:jupyter-init-script:write && yarn run package:copy-webview-toolkit && yarn run package:copy-markdown-it && yarn run package:copy-highlight && tsc --build --watch --verbose",
         "fix": "eslint src --fix && prettier . --write",
@@ -1917,6 +1926,7 @@
         "@types/shell-quote": "^1.7.5",
         "@vscode/debugadapter": "^1.64.0",
         "@vscode/extension-telemetry": "^1.5.2",
+        "@vscode/proxy-agent": "^0.44.0",
         "@vscode/webview-ui-toolkit": "^1.4.0",
         "add": "^2.0.6",
         "ansi-to-html": "^0.7.2",
@@ -1931,6 +1941,9 @@
         "winston": "^3.11.0",
         "yaml": "^2.8.3"
     },
+    "optionalDependencies": {
+        "@vscode/windows-ca-certs": "^0.3.1"
+    },
     "devDependencies": {
         "@istanbuljs/nyc-config-typescript": "^1.0.2",
         "@sinonjs/fake-timers": "^15.4.0",

packages/databricks-vscode/scripts/copy-windows-ca-certs.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copy the native @vscode/windows-ca-certs module into out/node_modules so the
+# esbuild bundle's runtime `require("@vscode/windows-ca-certs")` resolves in the
+# packaged extension. The bundle marks the module external (a native .node can't
+# be bundled) and the VSIX excludes the top-level node_modules/**, so the module
+# has to live under out/ to ship — same approach as the other package:copy-*
+# steps. It runs from vscode:prepublish (before vsce zips out/), which is the
+# only point at which we can add files to the packaged output.
+#
+# The module is a win32-only optional dependency, so this is a no-op unless we're
+# packaging a win32 VSIX. package-vsix.sh signals that by exporting
+# INCLUDE_WINDOWS_CA_CERTS=1 for the win32-* targets; for any other invocation
+# (dev builds, mac/linux VSIXs) we skip.
+#
+# Even for win32 targets the module may be absent when the VSIX is cross-built on
+# a non-Windows host (yarn honours the os=win32 install condition and skips it
+# there). We warn rather than fail: at runtime a missing module degrades to
+# Node's bundled roots plus databricks.proxy.caCert, so the build should still
+# succeed. To actually ship the native reader, build the win32 VSIX on a Windows
+# host (or install a prebuilt @vscode/windows-ca-certs) so the module is present.
+
+set -euo pipefail
+
+# get path of the repo (parent of scripts/)
+cd "$(dirname "$(realpath "$0")")/.."
+
+if [ "${INCLUDE_WINDOWS_CA_CERTS:-}" != "1" ]; then
+  # Not a win32 packaging target — nothing to do.
+  exit 0
+fi
+
+SRC="./node_modules/@vscode/windows-ca-certs"
+DEST="./out/node_modules/@vscode/windows-ca-certs"
+
+if [ ! -d "$SRC" ]; then
+  echo "WARNING: $SRC not found — packaging a win32 VSIX without the native" >&2
+  echo "Windows CA reader. On older Node the SDK will fall back to Node's" >&2
+  echo "bundled roots (+ databricks.proxy.caCert). Build on a Windows host to" >&2
+  echo "include it." >&2
+  exit 0
+fi
+
+rm -rf "$DEST"
+mkdir -p "$(dirname "$DEST")"
+# Copy the whole module directory (index.js, package.json, build/Release/*.node,
+# etc.) so its own require() calls and the native addon all resolve at runtime.
+cp -R "$SRC" "$DEST"
+
+echo "Copied @vscode/windows-ca-certs -> $DEST"

packages/databricks-vscode/scripts/package-vsix.sh

@@ -58,6 +58,14 @@ if [ $ARCH != "win32-arm64" ]; then
 fi
 
 yarn run prettier package.json --write
+
+# Ship the native Windows CA reader (@vscode/windows-ca-certs) only in the win32
+# VSIXs. vscode:prepublish reads this to decide whether to copy the module into
+# out/ (see scripts/copy-windows-ca-certs.sh).
+case $ARCH in
+  win32-*) export INCLUDE_WINDOWS_CA_CERTS=1 ;;
+esac
+
 TAG="release-v$(cat package.json | jq -r .version)" yarn run package -t $VSXI_ARCH
 
 git checkout -- package.json

packages/databricks-vscode/src/configuration/auth/AuthProvider.ts

@@ -1,22 +1,14 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import {
-    Config,
-    ProductVersion,
-    WorkspaceClient,
-    logging,
-} from "@databricks/sdk-experimental";
+import {Config, WorkspaceClient, logging} from "@databricks/sdk-experimental";
 import {CancellationToken, ProgressLocation, window} from "vscode";
 import {normalizeHost} from "../../utils/urlUtils";
 import {workspaceConfigs} from "../../vscode-objs/WorkspaceConfigs";
 
-// eslint-disable-next-line @typescript-eslint/no-require-imports
-const extensionVersion = require("../../../package.json")
-    .version as ProductVersion;
-
 import {AzureCliCheck} from "./AzureCliCheck";
 import {DatabricksCliCheck} from "./DatabricksCliCheck";
 import {Loggers} from "../../logger";
 import {CliWrapper} from "../../cli/CliWrapper";
+import {createWorkspaceClient} from "../../utils/network/proxyAgent";
 
 // TODO: Resolve this with SDK's AuthType.
 export type AuthType =
@@ -52,11 +44,7 @@ export abstract class AuthProvider {
 
     async getWorkspaceClient(): Promise<WorkspaceClient> {
         const config = await this.getSdkConfig();
-
-        return new WorkspaceClient(config, {
-            product: "databricks-vscode",
-            productVersion: extensionVersion,
-        });
+        return createWorkspaceClient(config, this.host);
     }
 
     /**

packages/databricks-vscode/src/configuration/auth/AzureCliCheck.ts

@@ -1,9 +1,4 @@
-import {
-    Context,
-    ProductVersion,
-    WorkspaceClient,
-    logging,
-} from "@databricks/sdk-experimental";
+import {Context, logging} from "@databricks/sdk-experimental";
 import {CancellationToken, commands, Disposable, Uri, window} from "vscode";
 import {Loggers} from "../../logger";
 import {AzureCliAuthProvider} from "./AuthProvider";
@@ -14,14 +9,11 @@ import {
     FileNotFoundException,
     isFileNotFound,
 } from "@databricks/sdk-experimental/dist/config/execUtils";
+import {createWorkspaceClient} from "../../utils/network/proxyAgent";
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 const {NamedLogger} = logging;
 
-// eslint-disable-next-line @typescript-eslint/no-require-imports
-const extensionVersion = require("../../../package.json")
-    .version as ProductVersion;
-
 type AzureCloud = "AzureCloud" | "AzureChinaCloud" | "AzureUSGovernment";
 
 type AzureStepName =
@@ -212,16 +204,13 @@ export class AzureCliCheck implements Disposable {
         expired: boolean;
         error?: Error;
     }> {
-        const workspaceClient = new WorkspaceClient(
+        const workspaceClient = await createWorkspaceClient(
             {
                 host: host.toString(),
                 authType: "azure-cli",
                 azureLoginAppId: this.azureLoginAppId,
             },
-            {
-                product: "databricks-vscode",
-                productVersion: extensionVersion,
-            }
+            host
         );
         try {
             await workspaceClient.currentUser.me(

packages/databricks-vscode/src/configuration/auth/DatabricksCliCheck.ts

@@ -1,8 +1,6 @@
 import {
     CancellationToken,
     Context,
-    ProductVersion,
-    WorkspaceClient,
     logging,
 } from "@databricks/sdk-experimental";
 import {
@@ -14,10 +12,7 @@ import {DatabricksCliAuthProvider} from "./AuthProvider";
 import {orchestrate, OrchestrationLoopError, Step} from "./orchestrate";
 import {Loggers} from "../../logger";
 import {execFile} from "../../cli/CliWrapper";
-
-// eslint-disable-next-line @typescript-eslint/no-require-imports
-const extensionVersion = require("../../../package.json")
-    .version as ProductVersion;
+import {createWorkspaceClient} from "../../utils/network/proxyAgent";
 
 type StepName = "tryLogin" | "login";
 
@@ -111,7 +106,7 @@ export class DatabricksCliCheck implements Disposable {
     private async tryLogin(
         cancellationToken?: CancellationToken
     ): Promise<boolean> {
-        const workspaceClient = new WorkspaceClient(
+        const workspaceClient = await createWorkspaceClient(
             {
                 host: this.authProvider.host.toString(),
                 authType: "databricks-cli",
@@ -121,10 +116,7 @@ export class DatabricksCliCheck implements Disposable {
                     ? {workspaceId: this.authProvider.workspaceId}
                     : {}),
             },
-            {
-                product: "databricks-vscode",
-                productVersion: extensionVersion,
-            }
+            this.authProvider.host
         );
 
         try {

packages/databricks-vscode/src/extension.ts

@@ -108,6 +108,7 @@ import {BundleVariableModel} from "./bundle/models/BundleVariableModel";
 import {BundleVariableTreeDataProvider} from "./ui/bundle-variables/BundleVariableTreeDataProvider";
 import {ConfigurationTreeViewManager} from "./ui/configuration-view/ConfigurationTreeViewManager";
 import {getCLIDependenciesEnvVars} from "./utils/envVarGenerators";
+import {applyProxyStrictSSLEnv} from "./utils/network/proxyAgent";
 import {EnvironmentCommands} from "./language/EnvironmentCommands";
 import {PackageManagerTelemetry} from "./language/PackageManagerTelemetry";
 import {WorkspaceFolderManager} from "./vscode-objs/WorkspaceFolderManager";
@@ -674,20 +675,16 @@ export async function activate(
         customWhenContext.updateShowClusterView();
     }
 
-    function updateStrictSSLEnv() {
-        const httpConfig = workspace.getConfiguration("http");
-        const proxyStrictSSL = httpConfig.get<boolean>("proxyStrictSSL");
-        process.env["DATABRICKS_SDK_PROXY_STRICT_SSL"] = proxyStrictSSL
-            ? "true"
-            : "false";
-    }
-
     updateFeatureContexts();
-    updateStrictSSLEnv();
+    // Resolve TLS strict-SSL through the same layered setting the SDK client
+    // path uses (databricks.proxy.strictSSL -> http.proxyStrictSSL -> true), so
+    // both writers of DATABRICKS_SDK_PROXY_STRICT_SSL agree and verification
+    // stays on by default when nothing is configured.
+    applyProxyStrictSSLEnv();
     context.subscriptions.push(
         workspace.onDidChangeConfiguration(() => {
             updateFeatureContexts();
-            updateStrictSSLEnv();
+            applyProxyStrictSSLEnv();
         })
     );
 

packages/databricks-vscode/src/utils/envVarGenerators.test.ts

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import {anything, instance, mock, when} from "ts-mockito";
+import {anything, instance, mock, reset, spy, when} from "ts-mockito";
 import {ConnectionManager} from "../configuration/ConnectionManager";
 import {DatabricksWorkspace} from "../configuration/DatabricksWorkspace";
 import {ApiClient, Config} from "@databricks/sdk-experimental";
@@ -13,6 +13,7 @@ import {
 import {Uri} from "vscode";
 import assert from "assert";
 import {AuthProvider} from "../configuration/auth/AuthProvider";
+import {workspaceConfigs} from "../vscode-objs/WorkspaceConfigs";
 
 describe(__filename, () => {
     let mockConnectionManager: ConnectionManager;
@@ -22,8 +23,14 @@ describe(__filename, () => {
     const mockHost = "http://example.com";
     let mockApiClient: ApiClient;
     let existingEnv: any;
+    let configsSpy: typeof workspaceConfigs;
 
     beforeEach(() => {
+        configsSpy = spy(workspaceConfigs);
+        // Default: no proxy configured via VS Code settings, so getProxyEnvVars
+        // falls back to the process env vars the tests set explicitly.
+        when(configsSpy.httpProxy).thenReturn(undefined);
+        when(configsSpy.httpNoProxy).thenReturn([]);
         mockConnectionManager = mock(ConnectionManager);
         mockDatabricksWorkspace = mock(DatabricksWorkspace);
         mockCluster = mock(Cluster);
@@ -45,6 +52,7 @@ describe(__filename, () => {
     });
 
     afterEach(() => {
+        reset(configsSpy);
         process.env = existingEnv;
     });
 
@@ -86,6 +94,51 @@ describe(__filename, () => {
         });
     });
 
+    it("should use the http.proxy setting for both schemes when set", () => {
+        delete process.env.HTTP_PROXY;
+        delete process.env.HTTPS_PROXY;
+        delete process.env.http_proxy;
+        delete process.env.https_proxy;
+        when(configsSpy.httpProxy).thenReturn("http://proxy.local:8080");
+
+        const actual = getProxyEnvVars();
+        assert.deepEqual(actual, {
+            HTTP_PROXY: "http://proxy.local:8080",
+            HTTPS_PROXY: "http://proxy.local:8080",
+            NO_PROXY: undefined,
+        });
+    });
+
+    it("should let the http.proxy setting win over env vars", () => {
+        process.env.HTTP_PROXY = "http://env-proxy.local:1";
+        process.env.HTTPS_PROXY = "http://env-proxy.local:1";
+        when(configsSpy.httpProxy).thenReturn("http://setting-proxy.local:2");
+
+        const actual = getProxyEnvVars();
+        assert.deepEqual(actual, {
+            HTTP_PROXY: "http://setting-proxy.local:2",
+            HTTPS_PROXY: "http://setting-proxy.local:2",
+            NO_PROXY: undefined,
+        });
+    });
+
+    it("should merge http.noProxy setting with the NO_PROXY env var", () => {
+        delete process.env.no_proxy;
+        process.env.NO_PROXY = "shared.example.com,env.example.com";
+        when(configsSpy.httpNoProxy).thenReturn([
+            "setting.example.com",
+            "shared.example.com",
+        ]);
+
+        const actual = getProxyEnvVars();
+        assert.deepEqual(actual, {
+            HTTP_PROXY: undefined,
+            HTTPS_PROXY: undefined,
+            // Deduped: "shared.example.com" appears in both sources but once here.
+            NO_PROXY: "setting.example.com,shared.example.com,env.example.com",
+        });
+    });
+
     it("should generate env vars with expected cluster id", () => {
         const actual = getCommonDatabricksEnvVars(
             instance(mockConnectionManager),