From bb6db62c640458db4d1a996ce9d5bc5b5cf52752 Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Sat, 17 Jan 2026 14:10:39 +1100 Subject: [PATCH 01/14] feat: Add adb-coding-assistants-cluster module Add Terraform module for deploying Databricks clusters pre-configured with Claude Code CLI for AI-assisted development. The module includes: - Unity Catalog Volume for secure init script storage - Automatic Claude Code CLI installation via init scripts - MLflow tracing integration for Claude sessions - Helper bash functions for cluster users - Support for single-node and autoscaling clusters Follows Databricks best practices using Unity Catalog Volumes for init script storage (DBR 13.3+). --- .../adb-coding-assistants-cluster/Makefile | 7 + .../adb-coding-assistants-cluster/README.md | 412 ++++++++++++++++++ modules/adb-coding-assistants-cluster/main.tf | 83 ++++ .../adb-coding-assistants-cluster/outputs.tf | 34 ++ .../scripts/README.md | 190 ++++++++ .../scripts/install-claude.sh | 362 +++++++++++++++ .../variables.tf | 94 ++++ .../adb-coding-assistants-cluster/versions.tf | 10 + 8 files changed, 1192 insertions(+) create mode 100644 modules/adb-coding-assistants-cluster/Makefile create mode 100644 modules/adb-coding-assistants-cluster/README.md create mode 100644 modules/adb-coding-assistants-cluster/main.tf create mode 100644 modules/adb-coding-assistants-cluster/outputs.tf create mode 100644 modules/adb-coding-assistants-cluster/scripts/README.md create mode 100644 modules/adb-coding-assistants-cluster/scripts/install-claude.sh create mode 100644 modules/adb-coding-assistants-cluster/variables.tf create mode 100644 modules/adb-coding-assistants-cluster/versions.tf diff --git a/modules/adb-coding-assistants-cluster/Makefile b/modules/adb-coding-assistants-cluster/Makefile new file mode 100644 index 00000000..653039d8 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/Makefile @@ -0,0 +1,7 @@ +.PHONY: docs test_docs + +docs: + terraform-docs -c ../../.terraform-docs.yml . + +test_docs: + terraform-docs -c ../../.terraform-docs.yml --output-check . diff --git a/modules/adb-coding-assistants-cluster/README.md b/modules/adb-coding-assistants-cluster/README.md new file mode 100644 index 00000000..71df1f5c --- /dev/null +++ b/modules/adb-coding-assistants-cluster/README.md @@ -0,0 +1,412 @@ +# Provisioning Databricks Cluster with Claude Code CLI + +This module deploys a Databricks cluster pre-configured with Claude Code CLI for AI-assisted development directly on Databricks. + +## Module content + +This module can be used to deploy the following: + +* Unity Catalog Volume for secure init script storage +* Init script with Claude Code CLI installation +* Databricks cluster with automatic AI coding assistant setup +* MLflow experiment configuration for tracing +* Helper bash functions for cluster users + +## Features + +- ✅ **Zero-configuration AI coding tools** on cluster startup +- ✅ **Unity Catalog Volumes** for secure script storage (Databricks recommended practice) +- ✅ **MLflow tracing** integration for Claude Code sessions +- ✅ **Flexible cluster configuration** (single-node or autoscaling) + +> **Note**: For offline/air-gapped environments, use the separate [`adb-coding-assistants-cluster-offline`](../adb-coding-assistants-cluster-offline/README.md) module. + +## Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ Unity Catalog Volume │ +│ /Volumes//// │ +│ └── install-claude.sh │ +└─────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────┐ +│ Databricks Cluster (on startup) │ +│ │ +│ 1. Executes init script from volume │ +│ 2. Installs Node.js, OpenCode, Claude CLI │ +│ 3. Configures bashrc with helper functions │ +│ 4. Auto-generates configs on user login │ +└─────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────┐ +│ User Login │ +│ │ +│ • DATABRICKS_TOKEN available from environment │ +│ • Configs auto-generate: │ +│ - ~/.claude/settings.json │ +│ - ~/.opencode/config.json │ +│ • Commands ready: claude, opencode │ +└─────────────────────────────────────────────────────┘ +``` + +## Prerequisites + +- Databricks workspace with Unity Catalog enabled +- Databricks Runtime 13.3 LTS or higher (recommended for Unity Catalog volumes) +- Databricks Terraform provider >= 1.40.0 +- Unity Catalog with an existing catalog and schema + +## Usage + +### Basic Example + +```hcl +module "coding_cluster" { + source = "./modules/coding-assistants-cluster" + + cluster_name = "ai-dev-cluster" + catalog_name = "main" + schema_name = "default" + + # init_script_source_path is optional - module includes the script +} +``` + +### Single-Node Cluster + +```hcl +module "single_node_cluster" { + source = "./modules/coding-assistants-cluster" + + cluster_name = "ai-dev-single-node" + catalog_name = "main" + schema_name = "default" + + cluster_mode = "SINGLE_NODE" + num_workers = 0 +} +``` + +### Autoscaling Cluster + +```hcl +module "autoscaling_cluster" { + source = "./modules/coding-assistants-cluster" + + cluster_name = "ai-dev-autoscaling" + catalog_name = "main" + schema_name = "default" + + min_workers = 2 + max_workers = 8 + + tags = { + Environment = "production" + Team = "data-science" + } +} +``` + +### Complete Example + +```hcl +module "coding_cluster" { + source = "./modules/coding-assistants-cluster" + + # Cluster configuration + cluster_name = "ai-development-cluster" + spark_version = "17.3.x-cpu-ml-scala2.13" + node_type_id = "Standard_D8pds_v6" + autotermination_minutes = 60 + + # Volume configuration + catalog_name = "main" + schema_name = "default" + volume_name = "coding_assistants" + + # Init script (optional - uses bundled script by default) + # init_script_source_path = "/path/to/custom/script.sh" + + # MLflow configuration + mlflow_experiment_name = "/Users/me@company.com/my-claude-traces" + + # Autoscaling + min_workers = 1 + max_workers = 5 + + # Tags + tags = { + Environment = "development" + Project = "ai-assisted-coding" + CostCenter = "engineering" + ManagedBy = "terraform" + } +} +``` + +## Init Script Storage Best Practices + +According to [Databricks documentation](https://docs.databricks.com/aws/en/init-scripts/): + +> **Databricks Runtime 13.3 LTS and above with Unity Catalog** +> Store init scripts in Unity Catalog volumes. + +### Why Unity Catalog Volumes? + +1. **Governance**: Full Unity Catalog ACL support +2. **Security**: Identity-based access control +3. **Portability**: Works across AWS, Azure, and GCP +4. **Versioning**: Easy to manage and update scripts +5. **No DBFS**: Recommended alternative to legacy DBFS storage + +### Init Script Identity + +- **Single-user access mode**: Uses assigned principal's identity +- **Standard access mode**: Uses cluster owner's identity +- **Volume access**: Governed by Unity Catalog permissions + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [databricks](#requirement\_databricks) | >= 1.40.0 | + +## Providers + +| Name | Version | +|------|---------| +| [databricks](#provider\_databricks) | 1.102.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [databricks_cluster.coding_assistants](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster) | resource | +| [databricks_file.init_script](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/file) | resource | +| [databricks_volume.init_scripts](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/volume) | resource | +| [databricks_current_user.me](https://registry.terraform.io/providers/databricks/databricks/latest/docs/data-sources/current_user) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [catalog\_name](#input\_catalog\_name) | Unity Catalog name for the volume | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Name of the Databricks cluster | `string` | n/a | yes | +| [autotermination\_minutes](#input\_autotermination\_minutes) | Minutes of inactivity before cluster auto-terminates | `number` | `30` | no | +| [cluster\_mode](#input\_cluster\_mode) | Cluster mode: STANDARD or SINGLE\_NODE | `string` | `"STANDARD"` | no | +| [init\_script\_source\_path](#input\_init\_script\_source\_path) | Local path to the init script | `string` | `null` | no | +| [max\_workers](#input\_max\_workers) | Maximum number of workers for autoscaling | `number` | `3` | no | +| [min\_workers](#input\_min\_workers) | Minimum number of workers for autoscaling | `number` | `1` | no | +| [mlflow\_experiment\_name](#input\_mlflow\_experiment\_name) | MLflow experiment name for Claude Code tracing | `string` | `"/Workspace/Shared/claude-code-tracing"` | no | +| [node\_type\_id](#input\_node\_type\_id) | Node type for the cluster. Default is Standard_D8pds_v6 (modern, premium SSD + local NVMe). If unavailable in your region, consider Standard_DS13_v2 as fallback. | `string` | `"Standard_D8pds_v6"` | no | +| [num\_workers](#input\_num\_workers) | Number of worker nodes (null for autoscaling) | `number` | `null` | no | +| [schema\_name](#input\_schema\_name) | Schema name for the volume | `string` | `"default"` | no | +| [spark\_version](#input\_spark\_version) | Databricks Runtime version | `string` | `"17.3.x-cpu-ml-scala2.13"` | no | +| [tags](#input\_tags) | Custom tags for the cluster | `map(string)` | 
{
  "Environment": "dev",
  "Purpose": "coding-assistants"
}
| no | +| [volume\_name](#input\_volume\_name) | Volume name to store init scripts | `string` | `"coding_assistants"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | The ID of the created cluster | +| [cluster\_name](#output\_cluster\_name) | Name of the created cluster | +| [cluster\_url](#output\_cluster\_url) | URL to access the cluster in Databricks UI | +| [init\_script\_path](#output\_init\_script\_path) | Path to the init script in the volume | +| [mlflow\_experiment\_name](#output\_mlflow\_experiment\_name) | MLflow experiment name for tracing | +| [volume\_full\_name](#output\_volume\_full\_name) | Full name of the volume | +| [volume\_path](#output\_volume\_path) | Path to the volume containing init scripts | + + +## Post-Deployment Usage + +### On the Cluster + +After the cluster starts, users can: + +```bash +# Check installation status +check-coding-assistants + +# Debug Claude configuration +claude-debug + +# Use Claude Code +claude "Analyze the customer churn data" + +# Use OpenCode +opencode "Generate unit tests for my functions" + +# Enable MLflow tracing +claude-tracing-enable + +# Check tracing status +claude-tracing-status +``` + +### Helper Commands + +The init script installs these helper commands in `~/.bashrc`: + +| Command | Purpose | +|---------|---------| +| `check-coding-assistants` | Verify installation and configuration | +| `claude-debug` | Show detailed Claude CLI configuration | +| `claude-refresh-token` | Regenerate Claude settings | +| `opencode-refresh-config` | Regenerate OpenCode config | +| `claude-tracing-enable` | Enable MLflow tracing | +| `claude-tracing-status` | Check tracing status | +| `claude-tracing-disable` | Disable MLflow tracing | + +## Cluster Access Modes + +### Single-User Access Mode + +```hcl +# Automatically configured by the module +data_security_mode = "SINGLE_USER" +single_user_name = data.databricks_current_user.me.user_name +``` + +### Standard Access Mode + +For standard access mode, you must: +1. Set up an allowlist for init scripts +2. Grant permissions to the volume + +See [Allowlist documentation](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/allowlist). + +## Troubleshooting + +### Init Script Fails + +Check cluster logs: +```bash +# Enable cluster log delivery in cluster config +# Then view: /cluster-logs//init_scripts/ +``` + +### Commands Not Found + +```bash +# Reload bashrc +source ~/.bashrc + +# Check PATH +echo $PATH | grep -E "(claude|opencode)" + +# Verify installation +check-coding-assistants +``` + +### Authentication Issues + +```bash +# Check environment variables +claude-debug + +# Verify token is set +echo $DATABRICKS_TOKEN + +# Regenerate configs +claude-refresh-token +opencode-refresh-config +``` + +### Script Size Limit + +Init scripts must be < 64KB. If exceeded: +- Break into multiple scripts +- Remove unnecessary comments +- Compress/optimize script + +## Security Considerations + +### Volume Permissions + +Ensure appropriate Unity Catalog permissions: + +```sql +-- Grant read access to volume +GRANT READ VOLUME ON VOLUME .. TO ; + +-- For standard access mode, add to allowlist +-- (Requires admin access) +``` + +### Token Security + +- Tokens are **never hardcoded** in configs +- Read from environment: `$DATABRICKS_TOKEN` +- Configs regenerate per session +- Settings files are user-readable only (`~/.claude/`, `~/.opencode/`) + +## Maintenance + +### Updating the Init Script + +1. Update the local init script file +2. Run `terraform apply` to upload new version +3. Restart clusters to apply changes + +```bash +terraform apply -target=module.coding_cluster.databricks_file.init_script +``` + +### Updating Cluster Configuration + +```bash +# Update variables in your config +# Then apply +terraform apply + +# Restart cluster for changes to take effect +``` + +## Cost Optimization + +- Use `autotermination_minutes` to automatically shut down idle clusters +- Use single-node mode for development: `cluster_mode = "SINGLE_NODE"` +- Enable autoscaling to scale down during low usage +- Consider spot instances (if supported by your cloud provider) + +## Limitations + +- Init scripts must be < 64KB +- Init script failures cause cluster launch to fail +- Requires Databricks Runtime 13.3 LTS+ for Unity Catalog volumes +- Standard access mode requires admin-configured allowlist + +## References + +- [Databricks Init Scripts Documentation](https://docs.databricks.com/init-scripts/) +- [Unity Catalog Volumes](https://docs.databricks.com/volumes/) +- [Databricks Terraform Provider](https://registry.terraform.io/providers/databricks/databricks/latest/docs) +- [Cluster Configuration](https://docs.databricks.com/compute/configure) + +## License + +This module is provided as-is for use with Databricks workspaces. + +## Contributing + +To contribute improvements to this module: +1. Test changes in an isolated Databricks workspace +2. Run `terraform validate` and `terraform fmt` +3. Update documentation for any new variables or outputs +4. Submit pull request with clear description of changes + +## Support + +For issues related to: +- **Module**: Open an issue in this repository +- **Init Script**: See the init script documentation +- **Databricks Platform**: Contact Databricks support +- **Claude/OpenCode**: Contact Anthropic or OpenCode support respectively diff --git a/modules/adb-coding-assistants-cluster/main.tf b/modules/adb-coding-assistants-cluster/main.tf new file mode 100644 index 00000000..578c1a20 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/main.tf @@ -0,0 +1,83 @@ +# Data source to get current user +data "databricks_current_user" "me" {} + +# Local value for init script path +locals { + init_script_path = var.init_script_source_path != null ? var.init_script_source_path : "${path.module}/scripts/install-claude.sh" +} + +# Create or reference the volume for init scripts +resource "databricks_volume" "init_scripts" { + name = var.volume_name + catalog_name = var.catalog_name + schema_name = var.schema_name + volume_type = "MANAGED" + comment = "Volume for Claude Code CLI init scripts" + + lifecycle { + ignore_changes = [owner] + } +} + +# Upload the init script to the volume +resource "databricks_file" "init_script" { + source = local.init_script_path + path = "${databricks_volume.init_scripts.volume_path}/install-claude.sh" +} + +# Create the cluster with init script +resource "databricks_cluster" "coding_assistants" { + cluster_name = var.cluster_name + spark_version = var.spark_version + node_type_id = var.node_type_id + autotermination_minutes = var.autotermination_minutes + data_security_mode = "SINGLE_USER" + single_user_name = data.databricks_current_user.me.user_name + + # Autoscaling or fixed size + # Autoscaling is not supported for single-node clusters + dynamic "autoscale" { + for_each = var.cluster_mode == "STANDARD" && var.num_workers == null ? [1] : [] + content { + min_workers = var.min_workers + max_workers = var.max_workers + } + } + + # For single-node clusters, num_workers must be 0 (driver-only) + # For standard clusters, use the provided num_workers value + num_workers = var.cluster_mode == "SINGLE_NODE" ? 0 : var.num_workers + + # Single node configuration + # According to Databricks docs: single-node clusters run Spark locally with no worker nodes + spark_conf = var.cluster_mode == "SINGLE_NODE" ? { + "spark.databricks.cluster.profile" = "singleNode" + "spark.master" = "local[*]" + } : {} + + custom_tags = merge( + var.tags, + { + "ManagedBy" = "Terraform" + }, + var.cluster_mode == "SINGLE_NODE" ? { + "ResourceClass" = "SingleNode" + } : {} + ) + + # Environment variables for Claude Code CLI + spark_env_vars = { + MLFLOW_EXPERIMENT_NAME = var.mlflow_experiment_name + } + + # Init script configuration + init_scripts { + volumes { + destination = "${databricks_volume.init_scripts.volume_path}/install-claude.sh" + } + } + + depends_on = [ + databricks_file.init_script + ] +} diff --git a/modules/adb-coding-assistants-cluster/outputs.tf b/modules/adb-coding-assistants-cluster/outputs.tf new file mode 100644 index 00000000..c19dca1a --- /dev/null +++ b/modules/adb-coding-assistants-cluster/outputs.tf @@ -0,0 +1,34 @@ +output "cluster_id" { + description = "The ID of the created cluster" + value = databricks_cluster.coding_assistants.id +} + +output "cluster_url" { + description = "URL to access the cluster in Databricks UI" + value = databricks_cluster.coding_assistants.url +} + +output "cluster_name" { + description = "Name of the created cluster" + value = databricks_cluster.coding_assistants.cluster_name +} + +output "volume_path" { + description = "Path to the volume containing init scripts" + value = databricks_volume.init_scripts.volume_path +} + +output "volume_full_name" { + description = "Full name of the volume" + value = "${var.catalog_name}.${var.schema_name}.${var.volume_name}" +} + +output "init_script_path" { + description = "Path to the init script in the volume" + value = databricks_file.init_script.path +} + +output "mlflow_experiment_name" { + description = "MLflow experiment name for tracing" + value = var.mlflow_experiment_name +} diff --git a/modules/adb-coding-assistants-cluster/scripts/README.md b/modules/adb-coding-assistants-cluster/scripts/README.md new file mode 100644 index 00000000..c1fd1410 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/scripts/README.md @@ -0,0 +1,190 @@ +# Claude Code CLI Installation Scripts + +This directory contains installation scripts for Claude Code CLI on Databricks clusters. + +## Scripts Overview + +| Script | Purpose | Network Required | +|--------|---------|------------------| +| `install-claude.sh` | Online installation (default) | ✅ Yes | + +> **Note**: For offline/air-gapped installations, use the separate [`adb-coding-assistants-cluster-offline`](../adb-coding-assistants-cluster-offline/README.md) module. + +## Quick Start + +### Online Installation (Default) + +For clusters with internet access: + +```hcl +resource "databricks_cluster" "claude_cluster" { + cluster_name = "claude-coding-assistant" + spark_version = data.databricks_spark_version.latest_lts.id + node_type_id = "Standard_D8pds_v6" + autotermination_minutes = 60 + num_workers = 0 + + init_scripts { + dbfs { + destination = "dbfs:/init-scripts/install-claude.sh" + } + } +} +``` + + +## What Gets Installed + +The script installs: + +- ✅ **Node.js 20.x** - Required runtime for Claude CLI +- ✅ **Claude Code CLI** - AI coding assistant +- ✅ **MLflow** - For tracing Claude interactions +- ✅ **System tools** - curl, wget, git, jq +- ✅ **Bash helpers** - Convenience functions for using Claude + +## Helper Commands + +After installation, these commands are available: + +```bash +# Verify installation +check-claude + +# Show debug info +claude-debug + +# Refresh authentication +claude-refresh-token + +# Enable MLflow tracing +claude-tracing-enable + +# Check tracing status +claude-tracing-status + +# Disable tracing +claude-tracing-disable +``` + +## Usage Examples + +```bash +# Interactive mode +claude + +# One-shot query +echo "Write a Python function to reverse a string" | claude --print + +# From file +claude < prompt.txt + +# With streaming +claude --stream < task.md +``` + +## Internet Dependencies (Online Mode) + +The online installer requires access to: + +| Domain | Purpose | +|--------|---------| +| `claude.ai` | Claude CLI installer | +| `deb.nodesource.com` | Node.js repository | +| `*.ubuntu.com` | System packages | +| `pypi.org` / `files.pythonhosted.org` | Python packages | +| `registry.npmjs.org` | NPM packages | +| `${DATABRICKS_HOST}` | Databricks API endpoints | + +## Firewall Configuration + +If using a firewall, allow HTTPS (443) to these domains, or use the offline installation method. + +## Environment Variables + +### Standard Variables (Set automatically by Databricks) + +- `DATABRICKS_HOST` - Workspace URL +- `DATABRICKS_TOKEN` - Authentication token + +### Optional Configuration + +- `MLFLOW_EXPERIMENT_NAME` - Custom experiment name (default: `/Workspace/Shared/claude-code-tracing`) + +## Architecture Support + +The installer supports: + +- ✅ **amd64** (x86_64) - Default +- ✅ **arm64** (aarch64) - Auto-detected + +## Troubleshooting + +### Installation fails during cluster startup + +Check the init script logs: +```bash +cat /tmp/init-script-claude.log +``` + +### Claude command not found + +Reload bashrc: +```bash +source ~/.bashrc +``` + +### Authentication errors + +Refresh token: +```bash +claude-refresh-token +``` + +### Installation works but Claude fails + +Check configuration: +```bash +check-claude +claude-debug +``` + +## File Structure + +``` +scripts/ +├── install-claude.sh # Online installer +└── README.md # This file +``` + +> **Offline Installation**: See the [`adb-coding-assistants-cluster-offline`](../adb-coding-assistants-cluster-offline/README.md) module for offline/air-gapped installation support. + +## Version Compatibility + +- **Databricks Runtime**: 13.0+ LTS recommended +- **Python**: 3.9+ (included in DBR) +- **Node.js**: 20.x (installed by script) +- **MLflow**: 3.4+ (installed by script) + +## Security Notes + +### Authentication +- Uses Databricks personal access tokens (auto-configured) +- Tokens are ephemeral and cluster-scoped +- No long-lived credentials stored + +### Network Security +- All traffic uses HTTPS +- Authentication via `ANTHROPIC_AUTH_TOKEN` environment variable +- Custom headers for Databricks integration + + +## Support + +- **Claude CLI Issues**: [Claude AI Documentation](https://claude.ai/docs) +- **Databricks Issues**: Contact Databricks Support +- **Script Issues**: Open issue in repository + +## License + +See repository LICENSE file. diff --git a/modules/adb-coding-assistants-cluster/scripts/install-claude.sh b/modules/adb-coding-assistants-cluster/scripts/install-claude.sh new file mode 100644 index 00000000..a461ca0b --- /dev/null +++ b/modules/adb-coding-assistants-cluster/scripts/install-claude.sh @@ -0,0 +1,362 @@ +#!/bin/bash +# +# Databricks Cluster Init Script - Claude Code CLI +# Installs Claude Code CLI with MLflow tracing +# +# Note: For offline/air-gapped installations, use the adb-coding-assistants-cluster-offline module instead +# + +set -uo pipefail +export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a CI=true + +L="/tmp/init-script-claude.log" +log() { echo "[$(date '+%H:%M:%S')] $1" | tee -a "$L"; } +cmd_exists() { command -v "$1" >/dev/null 2>&1; } + +# Install Claude Code CLI +install_claude() { + if cmd_exists claude; then + log "✓ Claude Code already installed" + return 0 + fi + + log "Installing Claude Code CLI..." + if curl -fsSL https://claude.ai/install.sh | bash &>>$L; then + log "✓ Claude Code installation completed" + return 0 + else + log "⚠ Claude Code installation failed (will be available after manual install)" + return 1 + fi +} + +# Install Node.js (required for Claude Code CLI) +install_nodejs() { + if cmd_exists node && cmd_exists npm; then + log "✓ Node.js already installed ($(node --version))" + return 0 + fi + + log "Installing Node.js 20.x..." + if curl -fsSL --max-time 300 --retry 3 https://deb.nodesource.com/setup_20.x | sudo -E bash - &>>$L; then + if sudo apt-get update -qq -y &>>$L && sudo apt-get install -y -qq nodejs &>>$L; then + if cmd_exists node && cmd_exists npm; then + log "✓ Node.js/npm installed successfully ($(node --version))" + return 0 + fi + fi + fi + + log "⚠ Node.js installation failed (Claude Code CLI will not work)" + return 1 +} + +# Add helper functions to bashrc +setup_bashrc() { + local START_MARKER="### CLAUDE_CODE_HELPERS_START ###" + local END_MARKER="### CLAUDE_CODE_HELPERS_END ###" + + # Backup bashrc + [ -f "$HOME/.bashrc" ] && cp "$HOME/.bashrc" "$HOME/.bashrc.backup-$(date +%s)" + + # Remove any existing Claude sections (between markers) + if [ -f "$HOME/.bashrc" ]; then + if grep -q "$START_MARKER" "$HOME/.bashrc" 2>/dev/null; then + log "Removing old bashrc helpers..." + # Remove everything between START and END markers (inclusive) + sed -i "/$START_MARKER/,/$END_MARKER/d" "$HOME/.bashrc" + fi + fi + + W="${DATABRICKS_HOST}" + E="${MLFLOW_EXPERIMENT_NAME:-/Workspace/Shared/claude-code-tracing}" + + log "Adding helpers to bashrc..." + + cat >> "$HOME/.bashrc" <<'EOF' + +### CLAUDE_CODE_HELPERS_START ### +# Claude Code CLI Setup (auto-generated - do not edit manually) +export PATH="$HOME/.claude/bin:$HOME/.local/bin:$PATH" + +# Claude Code MLflow tracing helpers +export DATABRICKS_HOST="${DATABRICKS_HOST:-WS_PH}" +export MLFLOW_EXPERIMENT_NAME="${MLFLOW_EXPERIMENT_NAME:-EXP_PH}" + +# Set Anthropic environment variables for Claude CLI +# NOTE: These env vars are the PRIMARY authentication method and take precedence +# over settings.json. They are always fresh because they're set on every login. +# The settings.json file serves as a fallback for cases where env vars aren't set. +# Using ANTHROPIC_AUTH_TOKEN only (not ANTHROPIC_API_KEY) to avoid auth conflicts. +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + export ANTHROPIC_AUTH_TOKEN="$DATABRICKS_TOKEN" + export ANTHROPIC_BASE_URL="${DATABRICKS_HOST}/serving-endpoints/anthropic" + export ANTHROPIC_MODEL="databricks-claude-sonnet-4-5" + export ANTHROPIC_CUSTOM_HEADERS="x-databricks-disable-beta-headers: true" +fi + +# Internal function to generate Claude settings (single source of truth) +_generate_claude_config() { + local config_file="$HOME/.claude/settings.json" + + cat > "$config_file" </dev/null 2>&1; then + if ! jq empty "$config_file" 2>/dev/null; then + echo "⚠ Claude settings JSON validation failed" >&2 + return 1 + fi + fi + + return 0 +} + +# Auto-generate Claude settings from environment on first login +# NOTE: settings.json acts as a FALLBACK - env vars (set above) are the primary method. +# This is only generated if the file doesn't exist, to provide authentication when +# env vars might not be present (e.g., in some non-standard shell environments). +if [ ! -f "$HOME/.claude/settings.json" ] && [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + mkdir -p "$HOME/.claude" + if _generate_claude_config; then + echo "✓ Claude Code settings.json created (fallback - env vars take precedence)" + else + echo "⚠ Failed to generate Claude settings (run claude-refresh-token to retry)" + fi +fi + +# Auto-enable Claude tracing on login (if not already enabled) +# This ensures tracing is always active and saves to the shared workspace path +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ] && command -v mlflow >/dev/null 2>&1; then + # Check if tracing is already enabled (non-zero exit means not enabled) + if ! mlflow autolog claude --status >/dev/null 2>&1; then + # Create experiment if it doesn't exist + python3 </dev/null 2>&1) && \ + echo "✓ Claude Code MLflow tracing auto-enabled in $WORKSPACE_DIR (experiment: EXP_PH)" + fi +fi + +# Regenerate Claude settings from current environment +claude-refresh-token() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + echo "⚠ DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + echo " On Databricks clusters, these should be automatically available" + return 1 + fi + + mkdir -p "$HOME/.claude" + _generate_claude_config + echo "✓ Claude Code settings updated with:" + echo " DATABRICKS_HOST: $DATABRICKS_HOST" + echo " DATABRICKS_TOKEN: \${DATABRICKS_TOKEN:0:20}..." +} + +claude-tracing-enable() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + echo "⚠ DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + echo " On Databricks clusters, these should be automatically available" + return 1 + fi + + if ! command -v mlflow >/dev/null 2>&1; then + echo "⚠ MLflow is not installed" + return 1 + fi + + # Create experiment if it doesn't exist + python3 </dev/null 2>&1; then + echo "✓ Claude Code CLI: $(which claude)" + claude --version 2>&1 | head -1 || echo " (version check failed)" + else + echo "✗ Claude Code CLI: not found" + [ -f "$HOME/.claude/bin/claude" ] && echo " Binary exists at: $HOME/.claude/bin/claude" + [ -f "$HOME/.local/bin/claude" ] && echo " Binary exists at: $HOME/.local/bin/claude" + fi + echo "" + + # Check configs + echo "Configuration files:" + if [ -f "$HOME/.claude/settings.json" ]; then + echo " ✓ Claude settings: $HOME/.claude/settings.json" + echo " Preview: $(head -3 $HOME/.claude/settings.json | tail -1)" + else + echo " ✗ Claude settings: missing" + fi + echo "" + + # Check environment + echo "Environment variables:" + [ -n "$DATABRICKS_HOST" ] && echo " ✓ DATABRICKS_HOST: ${DATABRICKS_HOST}" || echo " ✗ DATABRICKS_HOST: not set" + [ -n "$DATABRICKS_TOKEN" ] && echo " ✓ DATABRICKS_TOKEN: ${DATABRICKS_TOKEN:0:20}..." || echo " ✗ DATABRICKS_TOKEN: not set" + [ -n "$ANTHROPIC_API_KEY" ] && echo " ✓ ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:0:20}..." || echo " ✗ ANTHROPIC_API_KEY: not set" + [ -n "$ANTHROPIC_AUTH_TOKEN" ] && echo " ✓ ANTHROPIC_AUTH_TOKEN: ${ANTHROPIC_AUTH_TOKEN:0:20}..." || echo " ✗ ANTHROPIC_AUTH_TOKEN: not set" + [ -n "$ANTHROPIC_BASE_URL" ] && echo " ✓ ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL}" || echo " ✗ ANTHROPIC_BASE_URL: not set" + [ -n "$ANTHROPIC_MODEL" ] && echo " ✓ ANTHROPIC_MODEL: ${ANTHROPIC_MODEL}" || echo " ✗ ANTHROPIC_MODEL: not set" + [ -n "$ANTHROPIC_CUSTOM_HEADERS" ] && echo " ✓ ANTHROPIC_CUSTOM_HEADERS: ${ANTHROPIC_CUSTOM_HEADERS}" || echo " ✗ ANTHROPIC_CUSTOM_HEADERS: not set" + echo "" + + # Check MLflow + if command -v mlflow >/dev/null 2>&1; then + echo "✓ MLflow: $(mlflow --version 2>&1)" + else + echo "✗ MLflow: not found" + fi + echo "" + + # Test Claude authentication + echo "Testing Claude CLI authentication:" + if command -v claude >/dev/null 2>&1; then + if [ -n "$ANTHROPIC_API_KEY" ] || [ -n "$ANTHROPIC_AUTH_TOKEN" ]; then + echo " ✓ Authentication configured via environment variables" + echo " Test with: echo 'what is 1+1?' | claude --print" + else + echo " ⚠ ANTHROPIC_API_KEY/ANTHROPIC_AUTH_TOKEN not set" + echo " Run: source ~/.bashrc" + fi + fi + echo "" + + echo "Run 'source ~/.bashrc' if commands are still not found" +} + +claude-debug() { + echo "=== Claude CLI Debug Info ===" + echo "" + echo "Settings file:" + [ -f "$HOME/.claude/settings.json" ] && cat "$HOME/.claude/settings.json" || echo " Missing!" + echo "" + echo "Environment:" + env | grep -E "ANTHROPIC|DATABRICKS" || echo " No relevant env vars" + echo "" + echo "Claude config directory:" + ls -la "$HOME/.claude/" 2>/dev/null || echo " Directory doesn't exist" +} +### CLAUDE_CODE_HELPERS_END ### +EOF + + sed -i "s|WS_PH|$W|g; s|EXP_PH|$E|g" "$HOME/.bashrc" + log "✓ Bashrc helpers added" + log " Experiment: $E" +} + +# Main installation +main() { + log "Starting installation..." + + # Install system dependencies (curl, git, jq - commonly used by Claude Code) + log "Installing system dependencies..." + if sudo apt-get update -qq -y &>>$L; then + if sudo apt-get install -y -qq curl git jq &>>$L; then + log "✓ System dependencies installed (curl, git, jq)" + else + log "⚠ Some system dependencies failed to install" + fi + else + log "⚠ apt-get update failed" + fi + + # Install MLflow with Databricks support + log "Installing MLflow with Databricks support..." + if pip install --quiet --upgrade "mlflow[databricks]>=3.4" &>>$L; then + log "✓ MLflow installed successfully" + else + log "⚠ MLflow installation failed (tracing features will not work)" + fi + + # Install tools (continue even if some fail) + install_nodejs || log "⚠ Node.js installation skipped or failed" + install_claude || log "⚠ Claude Code installation skipped or failed" + + # Configure tools + if setup_bashrc; then + log "✓ Bashrc configuration completed" + else + log "⚠ Bashrc configuration failed" + fi + + log "" + log "=== Installation Summary ===" + log "Installation complete. Full log: $L" + log "" + log "Next steps (on cluster login):" + log " 1. Run: source ~/.bashrc" + log " 2. Verify: check-claude" + log " 3. Use: claude command" + log "" + log "Helper commands:" + log " - check-claude: Verify installation status" + log " - claude-debug: Show Claude CLI configuration details" + log " - claude-refresh-token: Regenerate Claude settings" + log " - claude-tracing-enable/disable/status: Manage MLflow tracing" + return 0 +} + +main +exit 0 diff --git a/modules/adb-coding-assistants-cluster/variables.tf b/modules/adb-coding-assistants-cluster/variables.tf new file mode 100644 index 00000000..61a40093 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/variables.tf @@ -0,0 +1,94 @@ +variable "cluster_name" { + description = "Name of the Databricks cluster" + type = string +} + +variable "catalog_name" { + description = "Unity Catalog name for the volume" + type = string +} + +variable "schema_name" { + description = "Schema name for the volume" + type = string + default = "default" +} + +variable "volume_name" { + description = "Volume name to store init scripts" + type = string + default = "coding_assistants" +} + +variable "init_script_source_path" { + description = "Local path to the init script" + type = string + default = null +} + +variable "spark_version" { + description = "Databricks Runtime version" + type = string + default = "17.3.x-cpu-ml-scala2.13" +} + +variable "node_type_id" { + description = "Node type for the cluster. Default is Standard_D8pds_v6 (modern, premium SSD + local NVMe). If unavailable in your region, consider Standard_DS13_v2 as fallback." + type = string + default = "Standard_D4ds_v5" +} + +variable "autotermination_minutes" { + description = "Minutes of inactivity before cluster auto-terminates" + type = number + default = 30 +} + +variable "num_workers" { + description = "Number of worker nodes (null for autoscaling). For SINGLE_NODE clusters, this is automatically set to 0." + type = number + default = null +} + +variable "min_workers" { + description = "Minimum number of workers for autoscaling" + type = number + default = 1 +} + +variable "max_workers" { + description = "Maximum number of workers for autoscaling" + type = number + default = 3 +} + +variable "mlflow_experiment_name" { + description = "MLflow experiment name for Claude Code tracing" + type = string + default = "/Workspace/Shared/claude-code-tracing" +} + +variable "cluster_mode" { + description = <<-EOT + Cluster mode: STANDARD or SINGLE_NODE. + - STANDARD: Multi-node cluster with worker nodes (supports autoscaling) + - SINGLE_NODE: Single-node cluster with no worker nodes (driver-only, runs Spark locally). + For SINGLE_NODE clusters, num_workers is automatically set to 0 and autoscaling is disabled. + EOT + type = string + default = "STANDARD" + + validation { + condition = contains(["STANDARD", "SINGLE_NODE"], var.cluster_mode) + error_message = "cluster_mode must be either STANDARD or SINGLE_NODE" + } +} + +variable "tags" { + description = "Custom tags for the cluster" + type = map(string) + default = { + Environment = "dev" + Purpose = "coding-assistants" + } +} diff --git a/modules/adb-coding-assistants-cluster/versions.tf b/modules/adb-coding-assistants-cluster/versions.tf new file mode 100644 index 00000000..07223296 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + databricks = { + source = "databricks/databricks" + version = ">= 1.40.0" + } + } +} From 043b456b37d17764471075a8f5529a75692f439e Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Sat, 17 Jan 2026 14:10:42 +1100 Subject: [PATCH 02/14] feat: Add adb-coding-assistants-cluster-offline module Add Terraform module for deploying Databricks clusters with Claude Code CLI in air-gapped or restricted network environments. The module includes: - Offline installation scripts for Claude Code CLI - Dependency downloader for preparing offline packages - Unity Catalog Volume for secure script storage - MLflow tracing integration - Support for DBFS and workspace file storage Designed for environments without internet access or strict firewall policies. --- .../README.md | 311 ++++++++++++++++++ .../main.tf | 84 +++++ .../outputs.tf | 39 +++ .../scripts/OFFLINE-INSTALLATION.md | 220 +++++++++++++ .../scripts/download-offline-dependencies.sh | 100 ++++++ .../scripts/install-claude-offline.sh | 295 +++++++++++++++++ .../variables.tf | 95 ++++++ .../versions.tf | 10 + 8 files changed, 1154 insertions(+) create mode 100644 modules/adb-coding-assistants-cluster-offline/README.md create mode 100644 modules/adb-coding-assistants-cluster-offline/main.tf create mode 100644 modules/adb-coding-assistants-cluster-offline/outputs.tf create mode 100644 modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md create mode 100755 modules/adb-coding-assistants-cluster-offline/scripts/download-offline-dependencies.sh create mode 100755 modules/adb-coding-assistants-cluster-offline/scripts/install-claude-offline.sh create mode 100644 modules/adb-coding-assistants-cluster-offline/variables.tf create mode 100644 modules/adb-coding-assistants-cluster-offline/versions.tf diff --git a/modules/adb-coding-assistants-cluster-offline/README.md b/modules/adb-coding-assistants-cluster-offline/README.md new file mode 100644 index 00000000..59493ecb --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/README.md @@ -0,0 +1,311 @@ +# Provisioning Databricks Cluster with Claude Code CLI (Offline Installation) + +This module deploys a Databricks cluster pre-configured with Claude Code CLI for AI-assisted development in **air-gapped or restricted network environments**. + +## Module content + +This module can be used to deploy the following: + +* Unity Catalog Volume for secure init script storage +* Init script with Claude Code CLI offline installation +* Databricks cluster with automatic AI coding assistant setup +* MLflow experiment configuration for tracing +* Helper bash functions for cluster users + +## Features + +- ✅ **Zero-configuration AI coding tools** on cluster startup +- ✅ **Unity Catalog Volumes** for secure script storage (Databricks recommended practice) +- ✅ **MLflow tracing** integration for Claude Code sessions +- ✅ **Flexible cluster configuration** (single-node or autoscaling) +- ✅ **Offline/air-gapped installation** - no internet access required + +## Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ Unity Catalog Volume │ +│ /Volumes//// │ +│ └── install-claude-offline.sh │ +└─────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────┐ +│ Databricks Cluster (on startup) │ +│ │ +│ 1. Executes init script from volume │ +│ 2. Installs Node.js, Claude CLI from local packages│ +│ 3. Configures bashrc with helper functions │ +│ 4. Auto-generates configs on user login │ +└─────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────┐ +│ User Login │ +│ │ +│ • DATABRICKS_TOKEN available from environment │ +│ • Configs auto-generate: │ +│ - ~/.claude/settings.json │ +│ • Commands ready: claude │ +└─────────────────────────────────────────────────────┘ +``` + +## Prerequisites + +- Databricks workspace with Unity Catalog enabled +- Databricks Runtime 13.3 LTS or higher (recommended for Unity Catalog volumes) +- Databricks Terraform provider >= 1.40.0 +- Unity Catalog with an existing catalog and schema +- **Offline packages** prepared and uploaded to DBFS (see [Offline Installation Guide](scripts/OFFLINE-INSTALLATION.md)) + +## Usage + +### Basic Example + +```hcl +module "coding_cluster_offline" { + source = "./modules/adb-coding-assistants-cluster-offline" + + cluster_name = "ai-dev-cluster-offline" + catalog_name = "main" + schema_name = "default" + + # Path to offline packages (defaults to /dbfs/init-scripts/offline-packages) + offline_packages_path = "/dbfs/init-scripts/offline-packages" +} +``` + +### Complete Example + +```hcl +module "coding_cluster_offline" { + source = "./modules/adb-coding-assistants-cluster-offline" + + # Cluster configuration + cluster_name = "ai-development-cluster-offline" + spark_version = "14.3.x-scala2.12" + node_type_id = "Standard_DS3_v2" + autotermination_minutes = 60 + + # Volume configuration + catalog_name = "main" + schema_name = "default" + volume_name = "coding_assistants_offline" + + # Offline packages path + offline_packages_path = "/dbfs/init-scripts/offline-packages" + + # MLflow configuration + mlflow_experiment_name = "/Workspace/Shared/claude-code-tracing" + + # Autoscaling + min_workers = 1 + max_workers = 5 + + # Tags + tags = { + Environment = "development" + Project = "ai-assisted-coding" + CostCenter = "engineering" + ManagedBy = "terraform" + } +} +``` + +## Preparing Offline Packages + +Before using this module, you must prepare offline packages: + +1. **Download dependencies** (on a machine with internet access): + ```bash + cd modules/adb-coding-assistants-cluster-offline/scripts + ./download-offline-dependencies.sh + ``` + +2. **Upload to Databricks**: + ```bash + databricks fs cp -r offline-packages/ dbfs:/init-scripts/offline-packages/ + ``` + +3. **Configure the module** with `offline_packages_path` pointing to the uploaded location. + +See [scripts/OFFLINE-INSTALLATION.md](scripts/OFFLINE-INSTALLATION.md) for detailed instructions. + +## Init Script Storage Best Practices + +According to [Databricks documentation](https://docs.databricks.com/aws/en/init-scripts/): + +> **Databricks Runtime 13.3 LTS and above with Unity Catalog** +> Store init scripts in Unity Catalog volumes. + +### Why Unity Catalog Volumes? + +1. **Governance**: Full Unity Catalog ACL support +2. **Security**: Identity-based access control +3. **Portability**: Works across AWS, Azure, and GCP +4. **Versioning**: Easy to manage and update scripts +5. **No DBFS**: Recommended alternative to legacy DBFS storage + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [databricks](#requirement\_databricks) | >= 1.40.0 | + +## Providers + +| Name | Version | +|------|---------| +| [databricks](#provider\_databricks) | >= 1.40.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [databricks_cluster.coding_assistants](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster) | resource | +| [databricks_file.init_script](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/file) | resource | +| [databricks_volume.init_scripts](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/volume) | resource | +| [databricks_current_user.me](https://registry.terraform.io/providers/databricks/databricks/latest/docs/data-sources/current_user) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [catalog\_name](#input\_catalog\_name) | Unity Catalog name for the volume | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Name of the Databricks cluster | `string` | n/a | yes | +| [autotermination\_minutes](#input\_autotermination\_minutes) | Minutes of inactivity before cluster auto-terminates | `number` | `30` | no | +| [cluster\_mode](#input\_cluster\_mode) | Cluster mode: STANDARD or SINGLE\_NODE | `string` | `"STANDARD"` | no | +| [init\_script\_source\_path](#input\_init\_script\_source\_path) | Local path to the init script | `string` | `null` | no | +| [max\_workers](#input\_max\_workers) | Maximum number of workers for autoscaling | `number` | `3` | no | +| [min\_workers](#input\_min\_workers) | Minimum number of workers for autoscaling | `number` | `1` | no | +| [mlflow\_experiment\_name](#input\_mlflow\_experiment\_name) | MLflow experiment name for Claude Code tracing | `string` | `"/Workspace/Shared/claude-code-tracing"` | no | +| [node\_type\_id](#input\_node\_type\_id) | Node type for the cluster | `string` | `"Standard_DS3_v2"` | no | +| [num\_workers](#input\_num\_workers) | Number of worker nodes (null for autoscaling) | `number` | `null` | no | +| [offline\_packages\_path](#input\_offline\_packages\_path) | Path to offline packages directory (e.g., /dbfs/init-scripts/offline-packages). If not set, defaults to /dbfs/init-scripts/offline-packages | `string` | `null` | no | +| [schema\_name](#input\_schema\_name) | Schema name for the volume | `string` | `"default"` | no | +| [spark\_version](#input\_spark\_version) | Databricks Runtime version | `string` | `"14.3.x-scala2.12"` | no | +| [tags](#input\_tags) | Custom tags for the cluster | `map(string)` | 
{
  "Environment": "dev",
  "Purpose": "coding-assistants-offline"
}
| no | +| [volume\_name](#input\_volume\_name) | Volume name to store init scripts | `string` | `"coding_assistants"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | The ID of the created cluster | +| [cluster\_name](#output\_cluster\_name) | Name of the created cluster | +| [cluster\_url](#output\_cluster\_url) | URL to access the cluster in Databricks UI | +| [init\_script\_path](#output\_init\_script\_path) | Path to the init script in the volume | +| [mlflow\_experiment\_name](#output\_mlflow\_experiment\_name) | MLflow experiment name for tracing | +| [offline\_packages\_path](#output\_offline\_packages\_path) | Path to offline packages directory | +| [volume\_full\_name](#output\_volume\_full\_name) | Full name of the volume | +| [volume\_path](#output\_volume\_path) | Path to the volume containing init scripts | + + +## Post-Deployment Usage + +### On the Cluster + +After the cluster starts, users can: + +```bash +# Check installation status +check-claude + +# Debug Claude configuration +claude-debug + +# Use Claude Code +claude "Analyze the customer churn data" + +# Enable MLflow tracing +claude-tracing-enable + +# Check tracing status +claude-tracing-status +``` + +### Helper Commands + +The init script installs these helper commands in `~/.bashrc`: + +| Command | Purpose | +|---------|---------| +| `check-claude` | Verify installation and configuration | +| `claude-debug` | Show detailed Claude CLI configuration | +| `claude-refresh-token` | Regenerate Claude settings | +| `claude-tracing-enable` | Enable MLflow tracing | +| `claude-tracing-status` | Check tracing status | +| `claude-tracing-disable` | Disable MLflow tracing | + +## Troubleshooting + +### Offline Packages Not Found + +If the cluster fails to start with "Offline packages not found": + +1. Verify packages are uploaded: + ```bash + databricks fs ls dbfs:/init-scripts/offline-packages/ + ``` + +2. Check the path in cluster environment variables: + ```bash + # Should match OFFLINE_PACKAGES_PATH + echo $OFFLINE_PACKAGES_PATH + ``` + +3. Ensure the path is accessible from the cluster (DBFS mount point) + +### Init Script Fails + +Check cluster logs: +```bash +# Enable cluster log delivery in cluster config +# Then view: /cluster-logs//init_scripts/ +``` + +### Commands Not Found + +```bash +# Reload bashrc +source ~/.bashrc + +# Check PATH +echo $PATH | grep claude + +# Verify installation +check-claude +``` + +## Security Considerations + +### Volume Permissions + +Ensure appropriate Unity Catalog permissions: + +```sql +-- Grant read access to volume +GRANT READ VOLUME ON VOLUME .. TO ; +``` + +### Token Security + +- Tokens are **never hardcoded** in configs +- Read from environment: `$DATABRICKS_TOKEN` +- Configs regenerate per session +- Settings files are user-readable only (`~/.claude/`) + +## References + +- [Offline Installation Guide](scripts/OFFLINE-INSTALLATION.md) +- [Databricks Init Scripts Documentation](https://docs.databricks.com/init-scripts/) +- [Unity Catalog Volumes](https://docs.databricks.com/volumes/) +- [Databricks Terraform Provider](https://registry.terraform.io/providers/databricks/databricks/latest/docs) + +## License + +This module is provided as-is for use with Databricks workspaces. diff --git a/modules/adb-coding-assistants-cluster-offline/main.tf b/modules/adb-coding-assistants-cluster-offline/main.tf new file mode 100644 index 00000000..fcb66989 --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/main.tf @@ -0,0 +1,84 @@ +# Data source to get current user +data "databricks_current_user" "me" {} + +# Local value for init script path +locals { + init_script_path = var.init_script_source_path != null ? var.init_script_source_path : "${path.module}/scripts/install-claude-offline.sh" +} + +# Create or reference the volume for init scripts +resource "databricks_volume" "init_scripts" { + name = var.volume_name + catalog_name = var.catalog_name + schema_name = var.schema_name + volume_type = "MANAGED" + comment = "Volume for Claude Code CLI offline init scripts" + + lifecycle { + ignore_changes = [owner] + } +} + +# Upload the init script to the volume +resource "databricks_file" "init_script" { + source = local.init_script_path + path = "${databricks_volume.init_scripts.volume_path}/install-claude-offline.sh" +} + +# Create the cluster with init script +resource "databricks_cluster" "coding_assistants" { + cluster_name = var.cluster_name + spark_version = var.spark_version + node_type_id = var.node_type_id + autotermination_minutes = var.autotermination_minutes + data_security_mode = "SINGLE_USER" + single_user_name = data.databricks_current_user.me.user_name + + # Autoscaling or fixed size + dynamic "autoscale" { + for_each = var.num_workers == null ? [1] : [] + content { + min_workers = var.min_workers + max_workers = var.max_workers + } + } + + num_workers = var.num_workers + + # Single node configuration + spark_conf = var.cluster_mode == "SINGLE_NODE" ? { + "spark.databricks.cluster.profile" = "singleNode" + "spark.master" = "local[*]" + } : {} + + custom_tags = merge( + var.tags, + { + "ManagedBy" = "Terraform" + }, + var.cluster_mode == "SINGLE_NODE" ? { + "ResourceClass" = "SingleNode" + } : {} + ) + + # Environment variables for Claude Code CLI + spark_env_vars = merge( + { + MLFLOW_EXPERIMENT_NAME = var.mlflow_experiment_name + }, + var.offline_packages_path != null ? { + OFFLINE_PACKAGES_PATH = var.offline_packages_path + } : {} + ) + + # Init script configuration + init_scripts { + volumes { + destination = "${databricks_volume.init_scripts.volume_path}/install-claude-offline.sh" + } + } + + depends_on = [ + databricks_file.init_script + ] +} diff --git a/modules/adb-coding-assistants-cluster-offline/outputs.tf b/modules/adb-coding-assistants-cluster-offline/outputs.tf new file mode 100644 index 00000000..052478a1 --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/outputs.tf @@ -0,0 +1,39 @@ +output "cluster_id" { + description = "The ID of the created cluster" + value = databricks_cluster.coding_assistants.id +} + +output "cluster_url" { + description = "URL to access the cluster in Databricks UI" + value = databricks_cluster.coding_assistants.url +} + +output "cluster_name" { + description = "Name of the created cluster" + value = databricks_cluster.coding_assistants.cluster_name +} + +output "volume_path" { + description = "Path to the volume containing init scripts" + value = databricks_volume.init_scripts.volume_path +} + +output "volume_full_name" { + description = "Full name of the volume" + value = "${var.catalog_name}.${var.schema_name}.${var.volume_name}" +} + +output "init_script_path" { + description = "Path to the init script in the volume" + value = databricks_file.init_script.path +} + +output "mlflow_experiment_name" { + description = "MLflow experiment name for tracing" + value = var.mlflow_experiment_name +} + +output "offline_packages_path" { + description = "Path to offline packages directory" + value = var.offline_packages_path != null ? var.offline_packages_path : "/dbfs/init-scripts/offline-packages" +} diff --git a/modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md b/modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md new file mode 100644 index 00000000..af9d3674 --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md @@ -0,0 +1,220 @@ +# Offline Installation Guide for Claude Code CLI + +This guide explains how to install Claude Code CLI on Databricks clusters without internet access (air-gapped or firewalled environments). + +## Overview + +The offline installation process involves two phases: + +1. **Download Phase** (on machine with internet access) +2. **Installation Phase** (on air-gapped Databricks cluster) + +## Phase 1: Download Dependencies + +Run this on a machine with internet access: + +```bash +# Download all dependencies +bash download-offline-dependencies.sh + +# Optional: specify architecture (default: amd64) +bash download-offline-dependencies.sh arm64 + +# Create tarball for transfer +tar czf claude-offline-packages.tar.gz offline-packages/ +``` + +This will create an `offline-packages/` directory containing: + +``` +offline-packages/ +├── apt/ # System packages (curl, wget, git, jq) +├── python/ # Python wheels (MLflow and dependencies) +├── node/ # Node.js 20.x +├── claude/ # Claude Code CLI installer +└── manifest.txt # Package inventory +``` + +## Phase 2: Upload to Databricks + +### Option A: Using DBFS + +```bash +# Upload to DBFS +databricks fs cp -r offline-packages/ dbfs:/init-scripts/offline-packages/ + +# Or using tarball +databricks fs cp claude-offline-packages.tar.gz dbfs:/init-scripts/ +``` + +### Option B: Using Workspace Files + +```bash +# Upload via Databricks CLI +databricks workspace import-dir offline-packages/ /Workspace/Shared/init-scripts/offline-packages/ +``` + +### Option C: Using Azure Storage (for Azure Databricks) + +```bash +# Upload to storage account used by your workspace +az storage blob upload-batch \ + --account-name \ + --destination init-scripts \ + --source offline-packages/ +``` + +## Phase 3: Configure Cluster Init Script + +### Using DBFS Path + +```hcl +resource "databricks_cluster" "claude_cluster" { + # ... other configuration ... + + init_scripts { + dbfs { + destination = "dbfs:/init-scripts/install-claude-offline.sh" + } + } + + spark_env_vars = { + OFFLINE_PACKAGES_PATH = "/dbfs/init-scripts/offline-packages" + } +} +``` + +### Using Workspace Path + +```hcl +resource "databricks_cluster" "claude_cluster" { + # ... other configuration ... + + init_scripts { + workspace { + destination = "/Shared/init-scripts/install-claude-offline.sh" + } + } + + spark_env_vars = { + OFFLINE_PACKAGES_PATH = "/Workspace/Shared/init-scripts/offline-packages" + } +} +``` + +## Phase 4: Install on Cluster + +The offline installation script will automatically run during cluster startup. + +After the cluster starts, SSH into the cluster and verify: + +```bash +# Reload bashrc +source ~/.bashrc + +# Check installation +check-claude + +# Test Claude CLI +echo "what is 1+1?" | claude --print +``` + +## Troubleshooting + +### Missing Packages + +If packages are missing from the offline bundle: + +```bash +# On internet-connected machine, download specific package +cd offline-packages/python +pip download + +# Re-upload to DBFS +databricks fs cp .whl dbfs:/init-scripts/offline-packages/python/ +``` + +### Wrong Architecture + +If you get "wrong architecture" errors: + +```bash +# Download for correct architecture +bash download-offline-dependencies.sh arm64 # or amd64 + +# Re-upload packages +``` + +### Claude Installer Still Tries to Download + +The Claude installer may still attempt internet access. To fully offline install: + +1. On an internet-connected machine with same OS/architecture as cluster, install Claude +2. Copy the installed binary: + ```bash + # After installing Claude on test machine + cp ~/.local/bin/claude offline-packages/claude/claude-binary + cp -r ~/.claude offline-packages/claude/claude-config/ + ``` + +3. Modify `install-claude-offline.sh` to copy binary directly: + ```bash + # Add to install_claude_offline function: + if [ -f "$OFFLINE_PACKAGES_PATH/claude/claude-binary" ]; then + mkdir -p "$HOME/.local/bin" + cp "$OFFLINE_PACKAGES_PATH/claude/claude-binary" "$HOME/.local/bin/claude" + chmod +x "$HOME/.local/bin/claude" + log "✓ Claude Code installed from offline binary" + return 0 + fi + ``` + +### Node.js Installation Fails + +If Node.js from .deb fails due to dependencies: + +```bash +# Use the tarball method instead - it's dependency-free +# The script automatically tries this as fallback +``` + +## Size Estimates + +Approximate download sizes: + +- Node.js: ~30-40 MB +- APT packages: ~5-10 MB +- Python packages (MLflow): ~50-80 MB +- Claude installer: ~1 MB +- **Total: ~100-150 MB** + +## Security Considerations + +For high-security environments: + +1. **Verify checksums** of downloaded packages +2. **Scan packages** for vulnerabilities before upload +3. **Use internal artifact repository** (Artifactory, Nexus) +4. **Sign packages** if required by your organization +5. **Maintain version inventory** for compliance + +## Alternative: Internal Mirror + +For large-scale deployments, consider setting up internal mirrors: + +```bash +# Example: Internal PyPI mirror +pip install --index-url https://pypi.internal.company.com/simple mlflow[databricks] + +# Example: Internal NPM registry +npm config set registry https://npm.internal.company.com +``` + +Then modify the online installer to use internal mirrors instead of public repositories. + +## Support + +For issues specific to: +- **Claude CLI**: See [Claude AI Documentation](https://claude.ai/docs) +- **Databricks clusters**: Contact Databricks support +- **This script**: Open an issue in the repository diff --git a/modules/adb-coding-assistants-cluster-offline/scripts/download-offline-dependencies.sh b/modules/adb-coding-assistants-cluster-offline/scripts/download-offline-dependencies.sh new file mode 100755 index 00000000..5740328d --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/scripts/download-offline-dependencies.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Download offline dependencies for Claude Code CLI installation +# Run this on a machine with internet access, then copy the offline-packages directory to your air-gapped environment +# + +set -euo pipefail + +OFFLINE_DIR="offline-packages" +ARCH="${1:-amd64}" # amd64 or arm64 + +echo "=== Downloading Offline Dependencies for Claude Code CLI ===" +echo "Architecture: $ARCH" +echo "Output directory: $OFFLINE_DIR" +echo "" + +# Create directory structure +mkdir -p "$OFFLINE_DIR"/{apt,python,node,claude} + +# Download Node.js packages +echo "[1/4] Downloading Node.js 20.x..." +cd "$OFFLINE_DIR/node" +if [ "$ARCH" = "amd64" ]; then + NODE_VERSION="20.11.1" + wget -q "https://deb.nodesource.com/node_20.x/pool/main/n/nodejs/nodejs_${NODE_VERSION}-1nodesource1_amd64.deb" || { + echo "⚠ Failed to download Node.js. Trying alternative method..." + # Alternative: download from official Node.js + wget -q "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" -O node-linux-x64.tar.xz + } +elif [ "$ARCH" = "arm64" ]; then + NODE_VERSION="20.11.1" + wget -q "https://deb.nodesource.com/node_20.x/pool/main/n/nodejs/nodejs_${NODE_VERSION}-1nodesource1_arm64.deb" || { + echo "⚠ Failed to download Node.js. Trying alternative method..." + wget -q "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-arm64.tar.xz" -O node-linux-arm64.tar.xz + } +fi +cd - > /dev/null +echo "✓ Node.js downloaded" + +# Download APT packages +echo "[2/4] Downloading APT packages (curl, wget, git, jq)..." +cd "$OFFLINE_DIR/apt" +apt-get download curl wget git jq 2>/dev/null || { + echo "⚠ apt-get download failed. Downloading manually..." + # Fallback: download from packages.ubuntu.com + UBUNTU_VERSION="jammy" # Ubuntu 22.04 + wget -q "http://archive.ubuntu.com/ubuntu/pool/main/c/curl/curl_7.81.0-1ubuntu1.15_${ARCH}.deb" 2>/dev/null || true + wget -q "http://archive.ubuntu.com/ubuntu/pool/main/w/wget/wget_1.21.2-2ubuntu1_${ARCH}.deb" 2>/dev/null || true + wget -q "http://archive.ubuntu.com/ubuntu/pool/main/g/git/git_2.34.1-1ubuntu1.10_${ARCH}.deb" 2>/dev/null || true + wget -q "http://archive.ubuntu.com/ubuntu/pool/universe/j/jq/jq_1.6-2.1ubuntu3_${ARCH}.deb" 2>/dev/null || true +} +cd - > /dev/null +echo "✓ APT packages downloaded" + +# Download Python packages +echo "[3/4] Downloading Python packages (MLflow)..." +cd "$OFFLINE_DIR/python" +pip download "mlflow[databricks]>=3.4" --dest . 2>/dev/null || { + echo "⚠ pip download failed. Make sure pip is installed." +} +cd - > /dev/null +echo "✓ Python packages downloaded" + +# Download Claude CLI +echo "[4/4] Downloading Claude Code CLI installer..." +cd "$OFFLINE_DIR/claude" +wget -q https://claude.ai/install.sh -O install.sh || { + echo "⚠ Failed to download Claude installer" +} +chmod +x install.sh +cd - > /dev/null +echo "✓ Claude installer downloaded" + +# Create manifest +cat > "$OFFLINE_DIR/manifest.txt" </dev/null 2>&1; } + +# Path to offline packages (can be overridden via environment variable) +OFFLINE_PACKAGES_PATH="${OFFLINE_PACKAGES_PATH:-/dbfs/init-scripts/offline-packages}" + +# Validate offline packages exist +if [ ! -d "$OFFLINE_PACKAGES_PATH" ]; then + log "✗ ERROR: Offline packages not found at: $OFFLINE_PACKAGES_PATH" + log " Please upload offline-packages directory to DBFS and set OFFLINE_PACKAGES_PATH" + exit 1 +fi + +log "Using offline packages from: $OFFLINE_PACKAGES_PATH" + +# Install Node.js from local package +install_nodejs_offline() { + if cmd_exists node && cmd_exists npm; then + log "✓ Node.js already installed ($(node --version))" + return 0 + fi + + log "Installing Node.js from offline packages..." + + # Try .deb package first + if ls "$OFFLINE_PACKAGES_PATH"/node/*.deb 1> /dev/null 2>&1; then + if sudo dpkg -i "$OFFLINE_PACKAGES_PATH"/node/*.deb &>>$L; then + log "✓ Node.js installed from .deb package" + return 0 + fi + fi + + # Try .tar.xz package + if ls "$OFFLINE_PACKAGES_PATH"/node/*.tar.xz 1> /dev/null 2>&1; then + local NODE_TAR=$(ls "$OFFLINE_PACKAGES_PATH"/node/*.tar.xz | head -1) + sudo tar -xJf "$NODE_TAR" -C /usr/local --strip-components=1 &>>$L + if cmd_exists node && cmd_exists npm; then + log "✓ Node.js installed from tarball" + return 0 + fi + fi + + log "⚠ Node.js installation failed" + return 1 +} + +# Install Claude Code CLI from local installer +install_claude_offline() { + if cmd_exists claude; then + log "✓ Claude Code already installed" + return 0 + fi + + log "Installing Claude Code CLI from offline package..." + + if [ -f "$OFFLINE_PACKAGES_PATH/claude/install.sh" ]; then + # Run the installer - it may still try to download, but at least we have it locally + if bash "$OFFLINE_PACKAGES_PATH/claude/install.sh" &>>$L; then + log "✓ Claude Code installation completed" + return 0 + fi + fi + + log "⚠ Claude Code installation failed" + return 1 +} + +# Install APT packages from local cache +install_apt_packages_offline() { + log "Installing system dependencies from offline packages..." + + if [ -d "$OFFLINE_PACKAGES_PATH/apt" ] && [ "$(ls -A $OFFLINE_PACKAGES_PATH/apt)" ]; then + # Install dependencies first to avoid dpkg errors + if sudo dpkg -i "$OFFLINE_PACKAGES_PATH"/apt/*.deb &>>$L; then + log "✓ System dependencies installed from offline packages" + return 0 + else + # Try to fix broken dependencies + sudo apt-get install -f -y &>>$L || true + log "⚠ Some packages may have failed to install" + fi + else + log "⚠ No APT packages found in offline directory" + fi + + return 0 +} + +# Install Python packages from local wheels +install_python_packages_offline() { + log "Installing MLflow from offline packages..." + + if [ -d "$OFFLINE_PACKAGES_PATH/python" ] && [ "$(ls -A $OFFLINE_PACKAGES_PATH/python)" ]; then + if pip install --no-index --find-links="$OFFLINE_PACKAGES_PATH/python" "mlflow[databricks]" &>>$L; then + log "✓ MLflow installed from offline packages" + return 0 + else + log "⚠ MLflow installation failed" + fi + else + log "⚠ No Python packages found in offline directory" + fi + + return 1 +} + +# Add helper functions to bashrc (same as online version) +setup_bashrc() { + local START_MARKER="### CLAUDE_CODE_HELPERS_START ###" + local END_MARKER="### CLAUDE_CODE_HELPERS_END ###" + + # Backup bashrc + [ -f "$HOME/.bashrc" ] && cp "$HOME/.bashrc" "$HOME/.bashrc.backup-$(date +%s)" + + # Remove any existing Claude sections (between markers) + if [ -f "$HOME/.bashrc" ]; then + if grep -q "$START_MARKER" "$HOME/.bashrc" 2>/dev/null; then + log "Removing old bashrc helpers..." + sed -i "/$START_MARKER/,/$END_MARKER/d" "$HOME/.bashrc" + fi + fi + + W="${DATABRICKS_HOST}" + E="${MLFLOW_EXPERIMENT_NAME:-/Workspace/Shared/claude-code-tracing}" + + log "Adding helpers to bashrc..." + + cat >> "$HOME/.bashrc" <<'EOF' + +### CLAUDE_CODE_HELPERS_START ### +# Claude Code CLI Setup (auto-generated - do not edit manually) +export PATH="$HOME/.claude/bin:$HOME/.local/bin:$PATH" + +# Claude Code MLflow tracing helpers +export DATABRICKS_HOST="${DATABRICKS_HOST:-WS_PH}" +export MLFLOW_EXPERIMENT_NAME="${MLFLOW_EXPERIMENT_NAME:-EXP_PH}" + +# Set Anthropic environment variables for Claude CLI +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + export ANTHROPIC_AUTH_TOKEN="$DATABRICKS_TOKEN" + export ANTHROPIC_BASE_URL="${DATABRICKS_HOST}/serving-endpoints/anthropic" + export ANTHROPIC_MODEL="databricks-claude-sonnet-4-5" + export ANTHROPIC_CUSTOM_HEADERS="x-databricks-disable-beta-headers: true" +fi + +# Internal function to generate Claude settings +_generate_claude_config() { + local config_file="$HOME/.claude/settings.json" + + cat > "$config_file" </dev/null 2>&1; then + echo "⚠ MLflow is not installed" + return 1 + fi + + python3 </dev/null 2>&1; then + echo "✓ Claude Code CLI: $(which claude)" + else + echo "✗ Claude Code CLI: not found" + fi + echo "" + [ -f "$HOME/.claude/settings.json" ] && echo "✓ Settings configured" || echo "✗ Settings missing" + [ -n "$ANTHROPIC_AUTH_TOKEN" ] && echo "✓ Authentication configured" || echo "✗ Authentication not set" +} + +claude-debug() { + echo "=== Claude CLI Debug Info ===" + [ -f "$HOME/.claude/settings.json" ] && cat "$HOME/.claude/settings.json" || echo "Settings missing!" + echo "" + env | grep -E "ANTHROPIC|DATABRICKS" || echo "No env vars" +} +### CLAUDE_CODE_HELPERS_END ### +EOF + + sed -i "s|WS_PH|$W|g; s|EXP_PH|$E|g" "$HOME/.bashrc" + log "✓ Bashrc helpers added" +} + +# Main installation +main() { + log "Starting offline installation..." + log "Offline packages: $OFFLINE_PACKAGES_PATH" + + # Install from offline packages + install_apt_packages_offline + install_python_packages_offline + install_nodejs_offline + install_claude_offline + + # Configure + if setup_bashrc; then + log "✓ Bashrc configuration completed" + fi + + log "" + log "=== Installation Summary ===" + log "Installation complete. Full log: $L" + log "" + log "Next steps (on cluster login):" + log " 1. Run: source ~/.bashrc" + log " 2. Verify: check-claude" + log " 3. Use: claude command" + log "" + log "Helper commands:" + log " - check-claude: Verify installation" + log " - claude-debug: Show configuration" + log " - claude-refresh-token: Update settings" + log " - claude-tracing-enable/disable/status: Manage tracing" + return 0 +} + +main +exit 0 diff --git a/modules/adb-coding-assistants-cluster-offline/variables.tf b/modules/adb-coding-assistants-cluster-offline/variables.tf new file mode 100644 index 00000000..4388d30a --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/variables.tf @@ -0,0 +1,95 @@ +variable "cluster_name" { + description = "Name of the Databricks cluster" + type = string +} + +variable "catalog_name" { + description = "Unity Catalog name for the volume" + type = string +} + +variable "schema_name" { + description = "Schema name for the volume" + type = string + default = "default" +} + +variable "volume_name" { + description = "Volume name to store init scripts" + type = string + default = "coding_assistants" +} + +variable "init_script_source_path" { + description = "Local path to the init script" + type = string + default = null +} + +variable "offline_packages_path" { + description = "Path to offline packages directory (e.g., /dbfs/init-scripts/offline-packages). If not set, defaults to /dbfs/init-scripts/offline-packages" + type = string + default = null +} + +variable "spark_version" { + description = "Databricks Runtime version" + type = string + default = "14.3.x-scala2.12" +} + +variable "node_type_id" { + description = "Node type for the cluster" + type = string + default = "Standard_DS3_v2" +} + +variable "autotermination_minutes" { + description = "Minutes of inactivity before cluster auto-terminates" + type = number + default = 30 +} + +variable "num_workers" { + description = "Number of worker nodes (null for autoscaling)" + type = number + default = null +} + +variable "min_workers" { + description = "Minimum number of workers for autoscaling" + type = number + default = 1 +} + +variable "max_workers" { + description = "Maximum number of workers for autoscaling" + type = number + default = 3 +} + +variable "mlflow_experiment_name" { + description = "MLflow experiment name for Claude Code tracing" + type = string + default = "/Workspace/Shared/claude-code-tracing" +} + +variable "cluster_mode" { + description = "Cluster mode: STANDARD or SINGLE_NODE" + type = string + default = "STANDARD" + + validation { + condition = contains(["STANDARD", "SINGLE_NODE"], var.cluster_mode) + error_message = "cluster_mode must be either STANDARD or SINGLE_NODE" + } +} + +variable "tags" { + description = "Custom tags for the cluster" + type = map(string) + default = { + Environment = "dev" + Purpose = "coding-assistants-offline" + } +} diff --git a/modules/adb-coding-assistants-cluster-offline/versions.tf b/modules/adb-coding-assistants-cluster-offline/versions.tf new file mode 100644 index 00000000..07223296 --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + databricks = { + source = "databricks/databricks" + version = ">= 1.40.0" + } + } +} From e2b1a2f64818864af4d3e5ebb80691853a23fdb6 Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Sat, 17 Jan 2026 14:10:44 +1100 Subject: [PATCH 03/14] feat: Add example for coding assistants cluster Add complete deployment example demonstrating usage of the adb-coding-assistants-cluster module. The example includes: - Azure Databricks workspace integration - Provider configuration with unified authentication - Complete variable definitions with examples - Post-deployment usage instructions - Troubleshooting guide Uses Databricks unified authentication supporting Azure CLI, environment variables, and configuration profiles. --- .../adb-coding-assistants-cluster/Makefile | 7 + .../adb-coding-assistants-cluster/README.md | 218 ++++++++++++++++++ .../adb-coding-assistants-cluster/main.tf | 21 ++ .../adb-coding-assistants-cluster/outputs.tf | 58 +++++ .../providers.tf | 53 +++++ .../terraform.tfvars.example | 70 ++++++ .../variables.tf | 94 ++++++++ .../adb-coding-assistants-cluster/versions.tf | 3 + 8 files changed, 524 insertions(+) create mode 100644 examples/adb-coding-assistants-cluster/Makefile create mode 100644 examples/adb-coding-assistants-cluster/README.md create mode 100644 examples/adb-coding-assistants-cluster/main.tf create mode 100644 examples/adb-coding-assistants-cluster/outputs.tf create mode 100644 examples/adb-coding-assistants-cluster/providers.tf create mode 100644 examples/adb-coding-assistants-cluster/terraform.tfvars.example create mode 100644 examples/adb-coding-assistants-cluster/variables.tf create mode 100644 examples/adb-coding-assistants-cluster/versions.tf diff --git a/examples/adb-coding-assistants-cluster/Makefile b/examples/adb-coding-assistants-cluster/Makefile new file mode 100644 index 00000000..653039d8 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/Makefile @@ -0,0 +1,7 @@ +.PHONY: docs test_docs + +docs: + terraform-docs -c ../../.terraform-docs.yml . + +test_docs: + terraform-docs -c ../../.terraform-docs.yml --output-check . diff --git a/examples/adb-coding-assistants-cluster/README.md b/examples/adb-coding-assistants-cluster/README.md new file mode 100644 index 00000000..7b3a5931 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/README.md @@ -0,0 +1,218 @@ +# Provisioning Databricks Cluster with Claude Code CLI + +This example uses the [adb-coding-assistants-cluster](../../modules/adb-coding-assistants-cluster) module. + +This template provides an example deployment of a Databricks cluster pre-configured with Claude Code CLI for AI-assisted development directly on the cluster. + +## What Gets Deployed + +* Unity Catalog Volume for init script storage +* Databricks cluster with Claude Code CLI auto-installed on startup +* MLflow experiment for tracing Claude Code sessions +* Bash helper functions for easy usage + +## How to use + +> **Note** +> A detailed module README with full configuration options can be found in [modules/adb-coding-assistants-cluster](../../modules/adb-coding-assistants-cluster) + +1. Reference this module using one of the different [module source types](https://developer.hashicorp.com/terraform/language/modules/sources) +2. Copy `terraform.tfvars.example` to `terraform.tfvars` +3. Update `terraform.tfvars` with your values: + - `databricks_resource_id`: Your Azure Databricks workspace resource ID + - `cluster_name`: Name for your cluster + - `catalog_name`: Unity Catalog name to use +4. (Optional) Customize cluster configuration in `terraform.tfvars` (node type, autoscaling, etc.) +5. (Optional) Configure your [remote backend](https://developer.hashicorp.com/terraform/language/settings/backends/azurerm) +6. Run `terraform init` to initialize terraform and get provider ready +7. Run `terraform plan` to review the resources that will be created +8. Run `terraform apply` to create the resources + +## Prerequisites + +- Databricks workspace with Unity Catalog enabled +- Unity Catalog with an existing catalog and schema +- Permission to create clusters +- (For Azure) Authenticated via `az login` or environment variables +- Databricks Runtime 14.3 LTS or higher recommended + +## Post-Deployment + +After the cluster starts, SSH or connect via notebook and run: + +```bash +# Reload bashrc to get helper commands +source ~/.bashrc + +# Verify installation +check-claude + +# Start using Claude +claude "Write a Python function to analyze customer churn" + +# Enable MLflow tracing (optional) +claude-tracing-enable +``` + +## Helper Commands + +| Command | Purpose | +|---------|---------| +| `check-claude` | Verify Claude CLI installation and configuration | +| `claude-debug` | Show detailed Claude configuration | +| `claude-refresh-token` | Regenerate Claude settings from environment | +| `claude-tracing-enable` | Enable MLflow tracing for Claude sessions | +| `claude-tracing-status` | Check tracing status | +| `claude-tracing-disable` | Disable tracing | + +## Offline Installation + +For air-gapped or restricted network environments, use the separate offline module: [`adb-coding-assistants-cluster-offline`](../../modules/adb-coding-assistants-cluster-offline/README.md). See the [Offline Installation Guide](../../modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md) for detailed instructions. + +## Configuration Examples + +### Single-Node Development Cluster + +```hcl +cluster_mode = "SINGLE_NODE" +num_workers = 0 +node_type_id = "Standard_D8pds_v6" +``` + +### Autoscaling Production Cluster + +```hcl +cluster_mode = "STANDARD" +num_workers = null # Enable autoscaling +min_workers = 2 +max_workers = 8 +node_type_id = "Standard_D8pds_v6" +``` + +## Authentication + +This example uses Databricks unified authentication. Authentication can be provided via: + +1. **Azure CLI** (recommended for local development): + ```bash + az login + terraform apply + ``` + +2. **Environment Variables** (recommended for CI/CD): + ```bash + export DATABRICKS_HOST="https://adb-xxx.azuredatabricks.net" + export DATABRICKS_TOKEN="dapi..." + terraform apply + ``` + +3. **Configuration Profile**: + ```bash + export DATABRICKS_CONFIG_PROFILE="my-profile" + terraform apply + ``` + +For more details on authentication, see the [Databricks unified authentication documentation](https://docs.databricks.com/dev-tools/auth/unified-auth.html). + +## Troubleshooting + +### Init Script Fails + +Check cluster event logs in the Databricks UI under **Compute** → **Your Cluster** → **Event Log**. + +Common issues: +- Network connectivity to download packages +- Unity Catalog volume permissions +- Insufficient cluster permissions + +### Claude Not Found After Login + +```bash +# Reload bashrc +source ~/.bashrc + +# Verify PATH +check-claude +``` + +### Authentication Issues + +```bash +# Check environment variables +check-claude + +# Regenerate configuration +claude-refresh-token +``` + +## Additional Resources + +- [Module Documentation](../../modules/adb-coding-assistants-cluster/README.md) +- [Offline Module Documentation](../../modules/adb-coding-assistants-cluster-offline/README.md) +- [Offline Installation Guide](../../modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md) +- [Scripts Documentation](../../modules/adb-coding-assistants-cluster/scripts/README.md) +- [Databricks Init Scripts Documentation](https://docs.databricks.com/clusters/init-scripts.html) +- [Unity Catalog Volumes Documentation](https://docs.databricks.com/data-governance/unity-catalog/volumes.html) + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [azurerm](#requirement\_azurerm) | >=4.31.0 | +| [databricks](#requirement\_databricks) | >=1.81.1 | + +## Providers + +| Name | Version | +|------|---------| +| [azurerm](#provider\_azurerm) | 4.57.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [claude\_cluster](#module\_claude\_cluster) | ../../modules/adb-coding-assistants-cluster | n/a | + +## Resources + +| Name | Type | +|------|------| +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_databricks_workspace.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/databricks_workspace) | data source | +| [azurerm_resource_group.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [catalog\_name](#input\_catalog\_name) | Unity Catalog name for the volume | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Name of the Databricks cluster | `string` | n/a | yes | +| [databricks\_resource\_id](#input\_databricks\_resource\_id) | The Azure resource ID for the Databricks workspace. Format: /subscriptions/{subscription-id}/resourceGroups/{resource-group}/providers/Microsoft.Databricks/workspaces/{workspace-name} | `string` | n/a | yes | +| [autotermination\_minutes](#input\_autotermination\_minutes) | Minutes of inactivity before cluster auto-terminates | `number` | `30` | no | +| [cluster\_mode](#input\_cluster\_mode) | Cluster mode: STANDARD or SINGLE\_NODE | `string` | `"STANDARD"` | no | +| [init\_script\_source\_path](#input\_init\_script\_source\_path) | Local path to the init script | `string` | `null` | no | +| [max\_workers](#input\_max\_workers) | Maximum number of workers for autoscaling | `number` | `3` | no | +| [min\_workers](#input\_min\_workers) | Minimum number of workers for autoscaling | `number` | `1` | no | +| [mlflow\_experiment\_name](#input\_mlflow\_experiment\_name) | MLflow experiment name for Claude Code tracing | `string` | `"/Workspace/Shared/claude-code-tracing"` | no | +| [node\_type\_id](#input\_node\_type\_id) | Node type for the cluster. Default is Standard_D8pds_v6 (modern, premium SSD + local NVMe). If unavailable in your region, consider Standard_DS13_v2 as fallback. | `string` | `"Standard_D8pds_v6"` | no | +| [num\_workers](#input\_num\_workers) | Number of worker nodes (null for autoscaling) | `number` | `null` | no | +| [schema\_name](#input\_schema\_name) | Schema name for the volume | `string` | `"default"` | no | +| [spark\_version](#input\_spark\_version) | Databricks Runtime version | `string` | `"17.3.x-cpu-ml-scala2.13"` | no | +| [tags](#input\_tags) | Custom tags for the cluster | `map(string)` | 
{
  "Environment": "dev",
  "Purpose": "coding-assistants"
}
| no | +| [volume\_name](#input\_volume\_name) | Volume name to store init scripts | `string` | `"coding_assistants"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | The ID of the created cluster | +| [cluster\_name](#output\_cluster\_name) | Name of the created cluster | +| [cluster\_url](#output\_cluster\_url) | URL to access the cluster in Databricks UI | +| [init\_script\_path](#output\_init\_script\_path) | Path to the init script in the volume | +| [mlflow\_experiment\_name](#output\_mlflow\_experiment\_name) | MLflow experiment name for tracing | +| [setup\_instructions](#output\_setup\_instructions) | Instructions for using the cluster | +| [volume\_full\_name](#output\_volume\_full\_name) | Full name of the volume | +| [volume\_path](#output\_volume\_path) | Path to the volume containing init scripts | + diff --git a/examples/adb-coding-assistants-cluster/main.tf b/examples/adb-coding-assistants-cluster/main.tf new file mode 100644 index 00000000..cd2519e6 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/main.tf @@ -0,0 +1,21 @@ +# Cluster with Claude Code CLI coding assistant +# Provider configuration is in providers.tf +module "claude_cluster" { + source = "../../modules/adb-coding-assistants-cluster" + + cluster_name = var.cluster_name + catalog_name = var.catalog_name + schema_name = var.schema_name + volume_name = var.volume_name + init_script_source_path = var.init_script_source_path + spark_version = var.spark_version + node_type_id = var.node_type_id + autotermination_minutes = var.autotermination_minutes + num_workers = var.num_workers + min_workers = var.min_workers + max_workers = var.max_workers + mlflow_experiment_name = var.mlflow_experiment_name + cluster_mode = var.cluster_mode + tags = var.tags +} + diff --git a/examples/adb-coding-assistants-cluster/outputs.tf b/examples/adb-coding-assistants-cluster/outputs.tf new file mode 100644 index 00000000..6c3cce86 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/outputs.tf @@ -0,0 +1,58 @@ +output "cluster_id" { + description = "The ID of the created cluster" + value = module.claude_cluster.cluster_id +} + +output "cluster_url" { + description = "URL to access the cluster in Databricks UI" + value = module.claude_cluster.cluster_url +} + +output "cluster_name" { + description = "Name of the created cluster" + value = module.claude_cluster.cluster_name +} + +output "volume_path" { + description = "Path to the volume containing init scripts" + value = module.claude_cluster.volume_path +} + +output "volume_full_name" { + description = "Full name of the volume" + value = module.claude_cluster.volume_full_name +} + +output "init_script_path" { + description = "Path to the init script in the volume" + value = module.claude_cluster.init_script_path +} + +output "mlflow_experiment_name" { + description = "MLflow experiment name for tracing" + value = module.claude_cluster.mlflow_experiment_name +} + +output "setup_instructions" { + description = "Instructions for using the cluster" + value = <<-EOT + Cluster deployed successfully! + + 1. Access cluster: ${module.claude_cluster.cluster_url} + 2. Wait for cluster to start (init script runs automatically) + 3. Open a notebook or terminal + 4. Run: source ~/.bashrc + 5. Verify: check-claude + 6. Start using: claude "your question" + + MLflow Experiment: ${module.claude_cluster.mlflow_experiment_name} + + Helper commands: + - check-claude: Verify installation status + - claude-debug: Show configuration details + - claude-refresh-token: Update authentication + - claude-tracing-enable: Enable MLflow tracing + - claude-tracing-status: Check tracing status + - claude-tracing-disable: Disable tracing + EOT +} diff --git a/examples/adb-coding-assistants-cluster/providers.tf b/examples/adb-coding-assistants-cluster/providers.tf new file mode 100644 index 00000000..906fe89d --- /dev/null +++ b/examples/adb-coding-assistants-cluster/providers.tf @@ -0,0 +1,53 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">=4.31.0" + } + databricks = { + source = "databricks/databricks" + version = ">=1.81.1" + } + } +} + +# Data source to get current Azure client configuration +data "azurerm_client_config" "current" {} + +# Extract subscription ID, resource group, and workspace name from resource ID +locals { + resource_regex = "(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft.Databricks/workspaces/(.+)" + subscription_id = regex(local.resource_regex, var.databricks_resource_id)[0] + resource_group = regex(local.resource_regex, var.databricks_resource_id)[1] + databricks_workspace_name = regex(local.resource_regex, var.databricks_resource_id)[2] +} + +# Data source to get the resource group +data "azurerm_resource_group" "this" { + name = local.resource_group +} + +# Configure the Azure Provider +provider "azurerm" { + subscription_id = local.subscription_id + features {} +} + +# Data source to get the Databricks workspace +data "azurerm_databricks_workspace" "this" { + name = local.databricks_workspace_name + resource_group_name = local.resource_group +} + +# Configure the Databricks Provider +# Authentication uses Databricks unified authentication: +# 1. Environment variables (DATABRICKS_HOST, DATABRICKS_TOKEN) - Recommended for CI/CD +# 2. Azure CLI authentication (az login) - Recommended for local development +# 3. Configuration profile (~/.databrickscfg) - Alternative for local development +# +# See: https://docs.databricks.com/dev-tools/auth/unified-auth.html +provider "databricks" { + host = data.azurerm_databricks_workspace.this.workspace_url + # No explicit authentication configured - uses unified authentication +} + diff --git a/examples/adb-coding-assistants-cluster/terraform.tfvars.example b/examples/adb-coding-assistants-cluster/terraform.tfvars.example new file mode 100644 index 00000000..25c6de60 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/terraform.tfvars.example @@ -0,0 +1,70 @@ +# Example terraform.tfvars file for Claude Code CLI Cluster +# Copy this to terraform.tfvars and customize for your environment + +# Required variables +databricks_resource_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Databricks/workspaces/my-workspace" +cluster_name = "claude-coding-assistant" +catalog_name = "main" + +# Optional variables with recommended defaults + +# Unity Catalog configuration +schema_name = "default" +volume_name = "coding_assistants" + +# Cluster configuration +spark_version = "17.3.x-cpu-ml-scala2.13" +node_type_id = "Standard_D8pds_v6" # Azure: Standard_D8pds_v6 (8 vCPU, 32 GB RAM, Premium SSD + local NVMe). Fallback: Standard_DS13_v2 if unavailable in region +autotermination_minutes = 30 + +# Cluster mode options: +# - "SINGLE_NODE": Cost-effective for individual development (recommended) +# - "STANDARD": Multi-node for team environments +cluster_mode = "SINGLE_NODE" + +# Worker configuration (ignored if cluster_mode = "SINGLE_NODE") +num_workers = 0 # Set to null to enable autoscaling, or a specific number + +# Autoscaling configuration (only used if num_workers = null) +min_workers = 1 +max_workers = 3 + +# MLflow experiment for Claude Code tracing +mlflow_experiment_name = "/Workspace/Shared/claude-code-tracing" + +# Optional: Custom init script path (defaults to bundled script) +# init_script_source_path = "./custom-install-claude.sh" + +# Custom tags +tags = { + Environment = "development" + Purpose = "ai-coding" + Owner = "data-engineering" + CostCenter = "engineering" +} + +# Cloud-specific node types: +# +# Azure VM Types (Premium SSD): +# Modern Dpdsv6-series (Cobalt 100 processor, Premium SSD + local NVMe): +# - Standard_D4pds_v6 (4 cores, 16 GB RAM) - Cost-effective +# - Standard_D8pds_v6 (8 cores, 32 GB RAM) - Recommended default (modern) +# - Standard_D16pds_v6 (16 cores, 64 GB RAM) - For larger workloads +# Note: Dpdsv6-series may have limited regional availability +# +# DS-series (Premium SSD, widely available): +# - Standard_DS3_v2 (4 cores, 14 GB RAM) - Cost-effective for development +# - Standard_DS4_v2 (8 cores, 28 GB RAM) - Good for medium workloads +# - Standard_DS13_v2 (8 cores, 56 GB RAM) - Good fallback if Dpdsv6 unavailable +# - Standard_DS5_v2 (16 cores, 56 GB RAM) - More CPU, same RAM as DS13_v2 +# - Standard_DS14_v2 (16 cores, 112 GB RAM) - For large-scale workloads +# +# AWS: +# - i3.xlarge (4 cores, 30.5 GB RAM) - Recommended for single-node +# - i3.2xlarge (8 cores, 61 GB RAM) - For larger workloads +# - r5.xlarge (4 cores, 32 GB RAM) - Memory-optimized +# +# GCP: +# - n1-highmem-4 (4 cores, 26 GB RAM) - Recommended for single-node +# - n1-highmem-8 (8 cores, 52 GB RAM) - For larger workloads +# - n2-standard-4 (4 cores, 16 GB RAM) - Cost-optimized diff --git a/examples/adb-coding-assistants-cluster/variables.tf b/examples/adb-coding-assistants-cluster/variables.tf new file mode 100644 index 00000000..b73fc922 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/variables.tf @@ -0,0 +1,94 @@ +variable "databricks_resource_id" { + description = "The Azure resource ID for the Databricks workspace. Format: /subscriptions/{subscription-id}/resourceGroups/{resource-group}/providers/Microsoft.Databricks/workspaces/{workspace-name}" + type = string +} + +variable "cluster_name" { + description = "Name of the Databricks cluster" + type = string +} + +variable "catalog_name" { + description = "Unity Catalog name for the volume" + type = string +} + +variable "schema_name" { + description = "Schema name for the volume" + type = string + default = "default" +} + +variable "volume_name" { + description = "Volume name to store init scripts" + type = string + default = "coding_assistants" +} + +variable "init_script_source_path" { + description = "Local path to the init script" + type = string + default = null +} + +variable "spark_version" { + description = "Databricks Runtime version" + type = string + default = "17.3.x-cpu-ml-scala2.13" +} + +variable "node_type_id" { + description = "Node type for the cluster. Default is Standard_D8pds_v6 (modern, premium SSD + local NVMe). If unavailable in your region, consider Standard_DS13_v2 as fallback." + type = string + default = "Standard_D8pds_v6" +} + +variable "autotermination_minutes" { + description = "Minutes of inactivity before cluster auto-terminates" + type = number + default = 30 +} + +variable "num_workers" { + description = "Number of worker nodes (null for autoscaling)" + type = number + default = null +} + +variable "min_workers" { + description = "Minimum number of workers for autoscaling" + type = number + default = 1 +} + +variable "max_workers" { + description = "Maximum number of workers for autoscaling" + type = number + default = 3 +} + +variable "mlflow_experiment_name" { + description = "MLflow experiment name for Claude Code tracing" + type = string + default = "/Workspace/Shared/claude-code-tracing" +} + +variable "cluster_mode" { + description = "Cluster mode: STANDARD or SINGLE_NODE" + type = string + default = "STANDARD" + + validation { + condition = contains(["STANDARD", "SINGLE_NODE"], var.cluster_mode) + error_message = "cluster_mode must be either STANDARD or SINGLE_NODE" + } +} + +variable "tags" { + description = "Custom tags for the cluster" + type = map(string) + default = { + Environment = "dev" + Purpose = "coding-assistants" + } +} diff --git a/examples/adb-coding-assistants-cluster/versions.tf b/examples/adb-coding-assistants-cluster/versions.tf new file mode 100644 index 00000000..7117131f --- /dev/null +++ b/examples/adb-coding-assistants-cluster/versions.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 1.0" +} From 3006f31ce36721d4f3e1ec3f2eae05fae6a8f350 Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Sat, 17 Jan 2026 14:10:46 +1100 Subject: [PATCH 04/14] docs: Update README and gitignore for new modules - Add adb-coding-assistants-cluster module to modules table - Add adb-coding-assistants-cluster-offline module to modules table - Add adb-coding-assistants-cluster example to examples table - Add *.plan pattern to .gitignore to prevent committing Terraform plan files --- .gitignore | 5 +++++ README.md | 2 ++ 2 files changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index a248d4ca..4f282ee0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ *.tfstate.lock. *.terraform.lock.hcl +# Terraform plan files +*.plan + # logs *.log @@ -22,6 +25,8 @@ # Ignored Terraform files *gitignore*.tf +terraform.tfvars +!terraform.tfvars.example # Ignore Mac .DS_Store files .DS_Store diff --git a/README.md b/README.md index 68baa2f6..8c0ad6f6 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ The folder `examples` contains the following Terraform implementation examples : | Azure | [adb-uc](examples/adb-uc/) | ADB Unity Catalog Process | | Azure | [adb-unity-catalog-basic-demo](examples/adb-unity-catalog-basic-demo/) | ADB Unity Catalog end-to-end demo including UC metastore setup, Users/groups sync from AAD to databricks account, UC Catalog, External locations, Schemas, & Access Grants | | Azure | [adb-overwatch](examples/adb-overwatch/) | Overwatch multi-workspace deployment on Azure | +| Azure | [adb-coding-assistants-cluster](examples/adb-coding-assistants-cluster/) | Databricks cluster with Claude Code CLI for AI-assisted development | | AWS | [aws-workspace-basic](examples/aws-workspace-basic/) | Provisioning AWS Databricks E2 | | AWS | [aws-workspace-with-firewall](examples/aws-workspace-with-firewall/) | Provisioning AWS Databricks E2 with an AWS Firewall | | AWS | [aws-exfiltration-protection](examples/aws-exfiltration-protection/) | An implementation of [Data Exfiltration Protection on AWS](https://www.databricks.com/blog/2021/02/02/data-exfiltration-protection-with-databricks-on-aws.html) | @@ -82,6 +83,7 @@ The folder `modules` contains the following Terraform modules : | Azure | [adb-overwatch-main-ws](modules/adb-overwatch-main-ws/) | Main Overwatch workspace deployment | | Azure | [adb-overwatch-ws-to-monitor](modules/adb-overwatch-ws-to-monitor/) | Overwatch deployment on the Azure workspace to monitor | | Azure | [adb-overwatch-analysis](modules/adb-overwatch-analysis/) | Overwatch analysis notebooks deployment on Azure | +| Azure | [adb-coding-assistants-cluster](modules/adb-coding-assistants-cluster/) | Databricks cluster with Claude Code CLI for AI-assisted development | | AWS | [aws-workspace-basic](modules/aws-workspace-basic/) | Provisioning AWS Databricks E2 | | AWS | [aws-databricks-base-infra](modules/aws-databricks-base-infra/) | Provisioning AWS Infrastructure to be used for the deployment of a Databricks E2 workspace | | AWS | [aws-databricks-unity-catalog](modules/aws-databricks-unity-catalog/) | Provisioning the AWS Infrastructure and setting up the metastore for Databricks Unity Catalog | From 60b034a7a62958ead6aca11e1018f59572c9ee88 Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Sat, 17 Jan 2026 14:58:45 +1100 Subject: [PATCH 05/14] docs: Add remote development guide to example README - Incorporate detailed usage instructions from temporary guide - Add steps for SSH setup, IDE connection, and port forwarding - Include tips for persistent sessions with tmux - Add instructions for finding Python interpreter path --- .../adb-coding-assistants-cluster/README.md | 88 +++++++++++++++++-- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/examples/adb-coding-assistants-cluster/README.md b/examples/adb-coding-assistants-cluster/README.md index 7b3a5931..bc6a17a3 100644 --- a/examples/adb-coding-assistants-cluster/README.md +++ b/examples/adb-coding-assistants-cluster/README.md @@ -38,22 +38,94 @@ This template provides an example deployment of a Databricks cluster pre-configu ## Post-Deployment -After the cluster starts, SSH or connect via notebook and run: +After the cluster starts, you can connect via SSH to use Claude Code and other development tools. + +### 1. Configure SSH Tunnel + +Use the Databricks CLI to set up SSH access to your new cluster: ```bash -# Reload bashrc to get helper commands -source ~/.bashrc +# Authenticate if needed +databricks auth login --host https://your-workspace-url.cloud.databricks.com -# Verify installation -check-claude +# Set up SSH config (replace 'claude-dev' with your preferred alias) +databricks ssh setup --name claude-dev +# Select your cluster from the list when prompted +``` + +This creates an entry in your `~/.ssh/config` file. + +### 2. Connect via VSCode or Cursor -# Start using Claude -claude "Write a Python function to analyze customer churn" +1. Install the **Remote - SSH** extension in VSCode or Cursor. +2. Open the Command Palette (`Cmd+Shift+P` / `Ctrl+Shift+P`). +3. Select **Remote-SSH: Connect to Host**. +4. Choose `claude-dev` (or the alias you created). +5. Select **Linux** as the platform. +6. Once connected, open your persistent workspace folder: `/Workspace/Users//`. -# Enable MLflow tracing (optional) +### 3. Launch Claude Code + +Open the terminal in your remote VSCode/Cursor session and run: + +```bash +# 1. Load environment variables and helpers +source ~/.bashrc + +# 2. Enable MLflow tracing (optional but recommended) claude-tracing-enable + +# 3. Start Claude Code +claude ``` +**First-time setup tips:** +- Claude will ask for file permissions; use `Shift+Tab` to auto-allow edits in the current directory. +- If you need to refresh credentials, run `claude-refresh-token`. + +### 4. Remote Web App Development (Port Forwarding) + +VSCode and Cursor automatically forward ports. For example, to run a Streamlit app: + +1. Create `app.py`: + ```python + import streamlit as st + st.title("Databricks Remote App") + st.write("Running on cluster!") + ``` +2. Run it: + ```bash + streamlit run app.py --server.port 8501 + ``` +3. Click "Open in Browser" in the popup notification to view it at `localhost:8501`. + +### 5. Using the Databricks Python Interpreter + +You don't need to configure a virtual environment. Databricks manages it for you. + +1. In the remote terminal, find the python path: + ```bash + echo $DATABRICKS_VIRTUAL_ENV + # Output example: /local_disk0/.ephemeral_nfs/envs/pythonEnv-xxxx/bin/python + ``` +2. In VSCode/Cursor, open the Command Palette and select **Python: Select Interpreter**. +3. Paste the path from above. + +### 6. Persistent Sessions with tmux + +To keep your agent running even if you disconnect: + +```bash +# Start a new session +tmux new -s claude-session + +# Detach (Ctrl+B, then D) +# Reattach later +tmux attach -t claude-session +``` + +This allows you to leave long-running tasks (like "Build a data pipeline") executing on the cluster while you are offline. + ## Helper Commands | Command | Purpose | From e19e47b4d35543d5a57d6a2985f2f48a5c9639de Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Sat, 17 Jan 2026 23:16:46 +1100 Subject: [PATCH 06/14] feat: Add token refresh and VS Code Remote SSH helpers Add automatic token refresh functionality and VS Code/Cursor Remote SSH setup helpers to both online and offline installation scripts. Token refresh features: - Automatic token refresh on shell login when token changes - Hourly cron job for periodic token refresh - Token hash tracking to detect changes efficiently - Helper commands: claude-token-status, claude-setup-token-refresh VS Code/Cursor Remote SSH features: - Helper functions to find Python virtual environment path - Setup guide and verification commands - VS Code settings.json generation - Standalone vscode-setup.sh script for independent use - Documentation updates with step-by-step instructions All helpers are automatically available after cluster initialization and support both VS Code and Cursor IDEs. --- .../README.md | 47 +++ .../scripts/install-claude-offline.sh | 378 +++++++++++++++++- .../scripts/vscode-setup.sh | 248 ++++++++++++ .../scripts/README.md | 107 +++++ .../scripts/install-claude.sh | 376 ++++++++++++++++- .../scripts/vscode-setup.sh | 248 ++++++++++++ 6 files changed, 1398 insertions(+), 6 deletions(-) create mode 100755 modules/adb-coding-assistants-cluster-offline/scripts/vscode-setup.sh create mode 100755 modules/adb-coding-assistants-cluster/scripts/vscode-setup.sh diff --git a/modules/adb-coding-assistants-cluster-offline/README.md b/modules/adb-coding-assistants-cluster-offline/README.md index 59493ecb..356f743a 100644 --- a/modules/adb-coding-assistants-cluster-offline/README.md +++ b/modules/adb-coding-assistants-cluster-offline/README.md @@ -240,6 +240,53 @@ The init script installs these helper commands in `~/.bashrc`: | `claude-tracing-enable` | Enable MLflow tracing | | `claude-tracing-status` | Check tracing status | | `claude-tracing-disable` | Disable MLflow tracing | +| `claude-vscode-setup` | Show VS Code/Cursor Remote SSH setup guide | +| `claude-vscode-env` | Get Python virtual environment path | +| `claude-vscode-check` | Verify VS Code/Cursor setup | +| `claude-vscode-config` | Generate VS Code settings.json snippet | +| `claude-token-status` | Check token freshness and auto-refresh status | +| `claude-setup-token-refresh` | Enable hourly automatic token refresh | +| `claude-remove-token-refresh` | Disable automatic token refresh | + +## VS Code/Cursor Remote SSH Setup + +For remote development using VS Code or Cursor, use the built-in helpers: + +```bash +# Show complete setup guide +claude-vscode-setup + +# Get Python interpreter path +claude-vscode-env + +# Verify setup +claude-vscode-check + +# Generate VS Code settings.json +claude-vscode-config +``` + +### Quick Setup Steps + +1. **Install Remote SSH Extension** + - VS Code: Install "Remote - SSH" extension + - Cursor: Built-in (already included) + +2. **Configure Default Extensions** + - Command Palette → `Remote-SSH: Settings` + - Add: `ms-Python.python` and `ms-toolsai.jupyter` + +3. **Connect to Cluster** + - Command Palette → `Remote-SSH: Connect to Host` + +4. **Select Python Interpreter** + - Run `claude-vscode-env` to get the path + - Command Palette → `Python: Select Interpreter` + - Use the `pythonEnv-xxx` interpreter for full Databricks Runtime access + +**Important**: Regular Python `.py` files don't have access to Databricks globals (`dbutils`, `spark`). Only IPYNB notebooks and Databricks notebooks have this access. + +See the [VS Code Setup Script](scripts/vscode-setup.sh) for a standalone helper. ## Troubleshooting diff --git a/modules/adb-coding-assistants-cluster-offline/scripts/install-claude-offline.sh b/modules/adb-coding-assistants-cluster-offline/scripts/install-claude-offline.sh index 1e8bd697..a8acf88d 100755 --- a/modules/adb-coding-assistants-cluster-offline/scripts/install-claude-offline.sh +++ b/modules/adb-coding-assistants-cluster-offline/scripts/install-claude-offline.sh @@ -172,9 +172,62 @@ _generate_claude_config() { } } CLAUDE_CONFIG + + # Validate JSON if jq is available + if command -v jq >/dev/null 2>&1; then + if ! jq empty "$config_file" 2>/dev/null; then + echo "⚠ Claude settings JSON validation failed" >&2 + return 1 + fi + fi + + # Store token hash for change detection + if [ -n "$DATABRICKS_TOKEN" ]; then + echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 > "$HOME/.claude/.token_hash" 2>/dev/null || true + fi + return 0 } +# Check if token has changed and refresh if needed +_check_and_refresh_token() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + return 0 # Skip if token not available + fi + + local config_file="$HOME/.claude/settings.json" + local token_hash_file="$HOME/.claude/.token_hash" + + # Calculate current token hash + local current_hash + current_hash=$(echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 2>/dev/null || echo "") + + if [ -z "$current_hash" ]; then + return 0 # Skip if hash calculation failed + fi + + # Check if token has changed + if [ -f "$token_hash_file" ]; then + local stored_hash + stored_hash=$(cat "$token_hash_file" 2>/dev/null || echo "") + if [ "$current_hash" = "$stored_hash" ]; then + return 0 # Token unchanged, no refresh needed + fi + fi + + # Token changed or first time - refresh config + mkdir -p "$HOME/.claude" + if _generate_claude_config >/dev/null 2>&1; then + # Only show message if in interactive shell (not cron) + if [ -t 0 ]; then + echo "✓ Claude Code token refreshed automatically" + fi + return 0 + fi + + return 1 +} + # Auto-generate Claude settings from environment on first login if [ ! -f "$HOME/.claude/settings.json" ] && [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then mkdir -p "$HOME/.claude" @@ -183,6 +236,12 @@ if [ ! -f "$HOME/.claude/settings.json" ] && [ -n "$DATABRICKS_TOKEN" ] && [ -n fi fi +# Auto-refresh token on shell login if it has changed +# This ensures settings.json stays in sync with DATABRICKS_TOKEN +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + _check_and_refresh_token +fi + # Regenerate Claude settings from current environment claude-refresh-token() { if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then @@ -191,8 +250,116 @@ claude-refresh-token() { fi mkdir -p "$HOME/.claude" - _generate_claude_config - echo "✓ Claude Code settings updated" + if _generate_claude_config; then + echo "✓ Claude Code settings updated" + else + echo "⚠ Failed to update Claude settings" + return 1 + fi +} + +# Setup cron job for periodic token refresh (runs hourly) +claude-setup-token-refresh() { + local cron_file="$HOME/.claude/token-refresh-cron" + + # Create cron wrapper script + mkdir -p "$HOME/.claude" + cat > "$cron_file" <<'CRON_SCRIPT' +#!/bin/bash +# Auto-generated cron script for Claude token refresh +# This script is called by cron to refresh the Claude token periodically + +# Source bashrc to get functions +if [ -f "$HOME/.bashrc" ]; then + source "$HOME/.bashrc" >/dev/null 2>&1 +fi + +# Check and refresh token if needed +_check_and_refresh_token +CRON_SCRIPT + chmod +x "$cron_file" + + # Check if cron job already exists + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + echo "✓ Token refresh cron job already configured" + return 0 + fi + + # Add cron job + (crontab -l 2>/dev/null; echo "0 * * * * $cron_file") | crontab - + if [ $? -eq 0 ]; then + echo "✓ Token refresh cron job configured (runs hourly)" + echo " To remove: crontab -e" + else + echo "⚠ Failed to setup cron job (may require cron service)" + return 1 + fi +} + +# Remove token refresh cron job +claude-remove-token-refresh() { + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + crontab -l 2>/dev/null | grep -v "token-refresh-cron" | crontab - + echo "✓ Token refresh cron job removed" + else + echo "ℹ No token refresh cron job found" + fi +} + +# Check token freshness status +claude-token-status() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + echo "⚠ DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + return 1 + fi + + local config_file="$HOME/.claude/settings.json" + local token_hash_file="$HOME/.claude/.token_hash" + + echo "=== Claude Token Status ===" + echo "" + + # Check if config file exists + if [ -f "$config_file" ]; then + echo "✓ Settings file: $config_file" + local file_age + file_age=$(stat -c %Y "$config_file" 2>/dev/null || stat -f %m "$config_file" 2>/dev/null || echo "0") + local current_time + current_time=$(date +%s) + local age_hours + age_hours=$(( (current_time - file_age) / 3600 )) + echo " Last updated: ${age_hours} hour(s) ago" + else + echo "✗ Settings file: missing" + fi + + echo "" + + # Check token hash + if [ -f "$token_hash_file" ]; then + local current_hash + current_hash=$(echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 2>/dev/null || echo "") + local stored_hash + stored_hash=$(cat "$token_hash_file" 2>/dev/null || echo "") + if [ "$current_hash" = "$stored_hash" ] && [ -n "$current_hash" ]; then + echo "✓ Token: matches stored hash (up to date)" + else + echo "⚠ Token: differs from stored hash (needs refresh)" + echo " Run: claude-refresh-token" + fi + else + echo "ℹ Token hash: not stored (will be created on next refresh)" + fi + + echo "" + + # Check cron job + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + echo "✓ Auto-refresh: enabled (hourly cron job)" + else + echo "ℹ Auto-refresh: disabled" + echo " Enable with: claude-setup-token-refresh" + fi } claude-tracing-enable() { @@ -243,6 +410,18 @@ check-claude() { echo "" [ -f "$HOME/.claude/settings.json" ] && echo "✓ Settings configured" || echo "✗ Settings missing" [ -n "$ANTHROPIC_AUTH_TOKEN" ] && echo "✓ Authentication configured" || echo "✗ Authentication not set" + echo "" + + # VS Code/Cursor Remote SSH info + echo "VS Code/Cursor Remote SSH:" + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo " ✓ Python virtual environment: $venv_path" + echo " Run 'claude-vscode-setup' for setup instructions" + else + echo " ℹ Run 'claude-vscode-setup' for Remote SSH setup guide" + fi } claude-debug() { @@ -251,6 +430,181 @@ claude-debug() { echo "" env | grep -E "ANTHROPIC|DATABRICKS" || echo "No env vars" } + +# VS Code/Cursor Remote SSH helpers +claude-vscode-env() { + # Show the Databricks virtual environment path for VS Code/Cursor + if [ -n "$DATABRICKS_VIRTUAL_ENV" ]; then + echo "$DATABRICKS_VIRTUAL_ENV" + else + # Try to find pythonEnv-* directories + local python_envs + python_envs=$(find /databricks/python* -maxdepth 1 -type d -name "pythonEnv-*" 2>/dev/null | head -1) + if [ -n "$python_envs" ]; then + echo "$python_envs" + else + echo "⚠ DATABRICKS_VIRTUAL_ENV not set and pythonEnv-* not found" + echo " Try: echo \$DATABRICKS_VIRTUAL_ENV" + return 1 + fi + fi +} + +claude-vscode-setup() { + echo "=== VS Code/Cursor Remote SSH Setup Guide ===" + echo "" + echo "1. Install Remote SSH Extension" + echo " - VS Code: Install 'Remote - SSH' extension" + echo " - Cursor: Built-in Remote SSH extension (already included)" + echo "" + echo "2. Configure Default Extensions" + echo " Open Command Palette (Cmd+Shift+P / Ctrl+Shift+P):" + echo " → Remote-SSH: Settings" + echo "" + echo " Or edit settings.json and add:" + echo "" + cat <<'VSCODE_SETTINGS' + "remote.SSH.defaultExtensions": [ + "ms-Python.python", + "ms-toolsai.jupyter" + ] +VSCODE_SETTINGS + echo "" + echo "3. Connect to Cluster" + echo " - Command Palette → Remote-SSH: Connect to Host" + echo " - Enter your cluster SSH connection details" + echo "" + echo "4. Select Python Interpreter" + echo " After connecting, run this command to get the Python path:" + echo "" + echo " $ claude-vscode-env" + echo "" + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo " Current virtual environment:" + echo " $venv_path" + echo "" + echo " Then in VS Code/Cursor:" + echo " - Command Palette → Python: Select Interpreter" + echo " - Paste the path above or browse to it" + else + echo " Run 'echo \$DATABRICKS_VIRTUAL_ENV' to find the path" + fi + echo "" + echo "5. Important Notes" + echo " • IPYNB notebooks and *.py Databricks notebooks have access to" + echo " Databricks globals (dbutils, spark, etc.)" + echo " • Regular Python *.py files do NOT have access to Databricks globals" + echo " • Always select the pythonEnv-xxx interpreter for full Databricks" + echo " Runtime library access" + echo "" + echo "6. Verify Setup" + echo " Run: claude-vscode-check" +} + +claude-vscode-check() { + echo "=== VS Code/Cursor Remote SSH Setup Check ===" + echo "" + + # Check for virtual environment + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "✓ Python Virtual Environment:" + echo " $venv_path" + if [ -d "$venv_path/bin" ]; then + echo " ✓ Virtual environment directory exists" + if [ -f "$venv_path/bin/python" ]; then + echo " ✓ Python executable found" + echo " Python version: $($venv_path/bin/python --version 2>&1 || echo 'unknown')" + else + echo " ⚠ Python executable not found" + fi + else + echo " ⚠ Virtual environment directory not found" + fi + else + echo "✗ Python Virtual Environment: Not found" + echo " Run: echo \$DATABRICKS_VIRTUAL_ENV" + fi + echo "" + + # Check for Python + if command -v python3 >/dev/null 2>&1; then + echo "✓ Python3 available: $(which python3)" + echo " Version: $(python3 --version 2>&1)" + else + echo "✗ Python3 not found in PATH" + fi + echo "" + + # Check for Databricks runtime libraries + echo "Databricks Runtime Libraries:" + python3 <<'PYTHON_CHECK' +import sys +libraries = ['pyspark', 'pandas', 'numpy', 'mlflow', 'databricks'] +found = [] +missing = [] + +for lib in libraries: + try: + __import__(lib) + found.append(lib) + except ImportError: + missing.append(lib) + +if found: + print(f" ✓ Available: {', '.join(found)}") +if missing: + print(f" ⚠ Missing: {', '.join(missing)}") + +# Check for Databricks globals (only available in notebooks) +try: + import dbutils + print(" ✓ dbutils available (notebook context)") +except: + print(" ℹ dbutils not available (normal for .py files)") +PYTHON_CHECK + + echo "" + echo "VS Code/Cursor Configuration:" + echo " Run 'claude-vscode-setup' for setup instructions" + echo " Run 'claude-vscode-env' to get Python interpreter path" +} + +claude-vscode-config() { + # Generate VS Code settings.json snippet + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + + echo "=== VS Code/Cursor settings.json Configuration ===" + echo "" + echo "Add this to your VS Code/Cursor settings.json:" + echo "" + echo "{" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "," + echo " \"python.defaultInterpreterPath\": \"$venv_path/bin/python\"" + fi + echo "}" + echo "" + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "Python interpreter path:" + echo " $venv_path/bin/python" + echo "" + echo "To set this in VS Code/Cursor:" + echo " 1. Command Palette → Python: Select Interpreter" + echo " 2. Enter interpreter path: $venv_path/bin/python" + else + echo "To find Python interpreter path, run:" + echo " claude-vscode-env" + fi +} ### CLAUDE_CODE_HELPERS_END ### EOF @@ -282,12 +636,32 @@ main() { log " 1. Run: source ~/.bashrc" log " 2. Verify: check-claude" log " 3. Use: claude command" + # Setup automatic token refresh (optional - user can enable manually) + log "Setting up automatic token refresh..." + if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + # Setup cron job for periodic refresh + if command -v crontab >/dev/null 2>&1; then + # Source bashrc temporarily to get the function + source "$HOME/.bashrc" >/dev/null 2>&1 || true + claude-setup-token-refresh >/dev/null 2>&1 || log "⚠ Cron setup skipped (may require manual setup)" + else + log "⚠ Cron not available - token refresh will only happen on login" + fi + fi + log "" log "Helper commands:" log " - check-claude: Verify installation" log " - claude-debug: Show configuration" log " - claude-refresh-token: Update settings" + log " - claude-token-status: Check token freshness and auto-refresh status" + log " - claude-setup-token-refresh: Enable hourly automatic token refresh" + log " - claude-remove-token-refresh: Disable automatic token refresh" log " - claude-tracing-enable/disable/status: Manage tracing" + log " - claude-vscode-setup: Show VS Code/Cursor Remote SSH setup guide" + log " - claude-vscode-env: Get Python virtual environment path" + log " - claude-vscode-check: Verify VS Code/Cursor setup" + log " - claude-vscode-config: Generate VS Code settings.json snippet" return 0 } diff --git a/modules/adb-coding-assistants-cluster-offline/scripts/vscode-setup.sh b/modules/adb-coding-assistants-cluster-offline/scripts/vscode-setup.sh new file mode 100755 index 00000000..89b21ada --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/scripts/vscode-setup.sh @@ -0,0 +1,248 @@ +#!/bin/bash +# +# VS Code/Cursor Remote SSH Setup Helper for Databricks Clusters +# This script helps configure VS Code or Cursor for remote development on Databricks clusters +# + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +log_info() { echo -e "${BLUE}ℹ${NC} $1"; } +log_success() { echo -e "${GREEN}✓${NC} $1"; } +log_warning() { echo -e "${YELLOW}⚠${NC} $1"; } +log_error() { echo -e "${RED}✗${NC} $1"; } + +# Find Databricks Python virtual environment +find_python_env() { + if [ -n "${DATABRICKS_VIRTUAL_ENV:-}" ]; then + echo "$DATABRICKS_VIRTUAL_ENV" + return 0 + fi + + # Try to find pythonEnv-* directories + local python_envs + python_envs=$(find /databricks/python* -maxdepth 1 -type d -name "pythonEnv-*" 2>/dev/null | head -1) + if [ -n "$python_envs" ]; then + echo "$python_envs" + return 0 + fi + + return 1 +} + +# Show setup instructions +show_setup_guide() { + echo "==========================================" + echo "VS Code/Cursor Remote SSH Setup Guide" + echo "==========================================" + echo "" + + echo "1. Install Remote SSH Extension" + echo " • VS Code: Install 'Remote - SSH' extension from marketplace" + echo " • Cursor: Built-in Remote SSH extension (already included)" + echo "" + + echo "2. Configure Default Extensions" + echo " Open Command Palette (Cmd+Shift+P / Ctrl+Shift+P):" + echo " → Type: Remote-SSH: Settings" + echo "" + echo " Or manually edit settings.json:" + echo "" + echo " {" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + echo " }" + echo "" + + echo "3. Connect to Cluster" + echo " • Command Palette → Remote-SSH: Connect to Host" + echo " • Enter your cluster SSH connection details" + echo " • Format: user@hostname or use SSH config entry" + echo "" + + echo "4. Select Python Interpreter" + local venv_path + if venv_path=$(find_python_env 2>/dev/null); then + echo " ✓ Found Python virtual environment:" + echo " $venv_path" + echo "" + echo " In VS Code/Cursor:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Enter interpreter path:" + echo " $venv_path/bin/python" + echo "" + echo " Or copy this path:" + echo " $venv_path/bin/python" + else + echo " ⚠ Could not auto-detect Python virtual environment" + echo " Run this command to find it:" + echo " echo \$DATABRICKS_VIRTUAL_ENV" + echo "" + echo " Then in VS Code/Cursor:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Paste the path from above" + fi + echo "" + + echo "5. Important Notes" + echo " • IPYNB notebooks and *.py Databricks notebooks have access to" + echo " Databricks globals (dbutils, spark, etc.)" + echo " • Regular Python *.py files do NOT have access to Databricks globals" + echo " • Always select the pythonEnv-xxx interpreter for full Databricks" + echo " Runtime library access (pyspark, pandas, numpy, mlflow, etc.)" + echo "" + + echo "6. Verify Setup" + echo " After connecting, verify Python interpreter:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Should show: pythonEnv-xxx/bin/python" + echo "" + echo " Test in a Python file:" + echo " import pyspark" + echo " import pandas" + echo " print('Setup successful!')" +} + +# Generate VS Code settings.json snippet +generate_settings() { + local venv_path + venv_path=$(find_python_env 2>/dev/null || echo "") + + echo "{" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + if [ -n "$venv_path" ]; then + echo "," + echo " \"python.defaultInterpreterPath\": \"$venv_path/bin/python\"" + fi + echo "}" +} + +# Check current setup +check_setup() { + echo "==========================================" + echo "VS Code/Cursor Setup Check" + echo "==========================================" + echo "" + + # Check for virtual environment + local venv_path + if venv_path=$(find_python_env 2>/dev/null); then + log_success "Python Virtual Environment found:" + echo " $venv_path" + + if [ -d "$venv_path/bin" ]; then + log_success "Virtual environment directory exists" + if [ -f "$venv_path/bin/python" ]; then + log_success "Python executable found" + echo " Python version: $($venv_path/bin/python --version 2>&1 || echo 'unknown')" + else + log_warning "Python executable not found" + fi + else + log_warning "Virtual environment directory not found" + fi + else + log_error "Python Virtual Environment not found" + echo " Run: echo \$DATABRICKS_VIRTUAL_ENV" + fi + echo "" + + # Check for Python + if command -v python3 >/dev/null 2>&1; then + log_success "Python3 available: $(which python3)" + echo " Version: $(python3 --version 2>&1)" + else + log_error "Python3 not found in PATH" + fi + echo "" + + # Check for Databricks runtime libraries + echo "Databricks Runtime Libraries:" + python3 <<'PYTHON_CHECK' +import sys +libraries = ['pyspark', 'pandas', 'numpy', 'mlflow'] +found = [] +missing = [] + +for lib in libraries: + try: + __import__(lib) + found.append(lib) + except ImportError: + missing.append(lib) + +if found: + print(f" ✓ Available: {', '.join(found)}") +if missing: + print(f" ⚠ Missing: {', '.join(missing)}") + +# Check for Databricks globals (only available in notebooks) +try: + import dbutils + print(" ✓ dbutils available (notebook context)") +except: + print(" ℹ dbutils not available (normal for .py files)") +PYTHON_CHECK + + echo "" + echo "Next steps:" + echo " • Run this script with --guide to see setup instructions" + echo " • Run this script with --settings to generate settings.json" +} + +# Main +main() { + case "${1:-}" in + --guide|-g) + show_setup_guide + ;; + --settings|-s) + generate_settings + ;; + --check|-c) + check_setup + ;; + --env|-e) + find_python_env || { + log_error "Could not find Python virtual environment" + echo "Try: echo \$DATABRICKS_VIRTUAL_ENV" + exit 1 + } + ;; + --help|-h|"") + echo "VS Code/Cursor Remote SSH Setup Helper" + echo "" + echo "Usage: $0 [OPTION]" + echo "" + echo "Options:" + echo " --guide, -g Show complete setup guide" + echo " --settings, -s Generate VS Code settings.json snippet" + echo " --check, -c Check current setup status" + echo " --env, -e Show Python virtual environment path" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 --guide # Show setup instructions" + echo " $0 --env # Get Python interpreter path" + echo " $0 --check # Verify setup" + ;; + *) + log_error "Unknown option: $1" + echo "Run '$0 --help' for usage information" + exit 1 + ;; + esac +} + +main "$@" diff --git a/modules/adb-coding-assistants-cluster/scripts/README.md b/modules/adb-coding-assistants-cluster/scripts/README.md index c1fd1410..7547724d 100644 --- a/modules/adb-coding-assistants-cluster/scripts/README.md +++ b/modules/adb-coding-assistants-cluster/scripts/README.md @@ -57,6 +57,11 @@ claude-debug # Refresh authentication claude-refresh-token +# Token management +claude-token-status # Check token freshness +claude-setup-token-refresh # Enable automatic hourly refresh +claude-remove-token-refresh # Disable automatic refresh + # Enable MLflow tracing claude-tracing-enable @@ -65,6 +70,108 @@ claude-tracing-status # Disable tracing claude-tracing-disable + +# VS Code/Cursor Remote SSH helpers +claude-vscode-setup # Show setup guide +claude-vscode-env # Get Python virtual environment path +claude-vscode-check # Verify VS Code/Cursor setup +claude-vscode-config # Generate VS Code settings.json snippet +``` + +## VS Code/Cursor Remote SSH Setup + +For remote development using VS Code or Cursor, follow these steps: + +### Quick Setup + +1. **Get Python interpreter path** (after SSH connection): + ```bash + claude-vscode-env + # Or manually: echo $DATABRICKS_VIRTUAL_ENV + ``` + +2. **Show complete setup guide**: + ```bash + claude-vscode-setup + ``` + +3. **Generate VS Code settings**: + ```bash + claude-vscode-config + ``` + +### Detailed Steps + +#### 1. Install Remote SSH Extension + +- **VS Code**: Install "Remote - SSH" extension from marketplace +- **Cursor**: Built-in Remote SSH extension (already included) + +#### 2. Configure Default Extensions + +Open Command Palette (`Cmd+Shift+P` / `Ctrl+Shift+P`): +- Type: `Remote-SSH: Settings` +- Or manually edit `settings.json`: + +```json +{ + "remote.SSH.defaultExtensions": [ + "ms-Python.python", + "ms-toolsai.jupyter" + ] +} +``` + +#### 3. Connect to Cluster + +- Command Palette → `Remote-SSH: Connect to Host` +- Enter your cluster SSH connection details + +#### 4. Select Python Interpreter + +After connecting: + +1. Run `claude-vscode-env` to get the Python path +2. Command Palette → `Python: Select Interpreter` +3. Enter or browse to: `/databricks/python*/pythonEnv-*/bin/python` + +**Important**: Always select the `pythonEnv-xxx` interpreter for full Databricks Runtime library access. + +#### 5. Verify Setup + +```bash +# Check setup status +claude-vscode-check + +# Test in a Python file +import pyspark +import pandas +import mlflow +print("Setup successful!") +``` + +### Important Notes + +- **IPYNB notebooks** and **`*.py` Databricks notebooks** have access to Databricks globals (`dbutils`, `spark`, etc.) +- **Regular Python `*.py` files** do NOT have access to Databricks globals +- Always select the `pythonEnv-xxx` interpreter for full Databricks Runtime library access + +### Standalone Helper Script + +A standalone helper script is also available: + +```bash +# Show setup guide +./scripts/vscode-setup.sh --guide + +# Get Python interpreter path +./scripts/vscode-setup.sh --env + +# Check current setup +./scripts/vscode-setup.sh --check + +# Generate settings.json +./scripts/vscode-setup.sh --settings ``` ## Usage Examples diff --git a/modules/adb-coding-assistants-cluster/scripts/install-claude.sh b/modules/adb-coding-assistants-cluster/scripts/install-claude.sh index a461ca0b..605f0292 100644 --- a/modules/adb-coding-assistants-cluster/scripts/install-claude.sh +++ b/modules/adb-coding-assistants-cluster/scripts/install-claude.sh @@ -118,9 +118,53 @@ CLAUDE_CONFIG fi fi + # Store token hash for change detection + if [ -n "$DATABRICKS_TOKEN" ]; then + echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 > "$HOME/.claude/.token_hash" 2>/dev/null || true + fi + return 0 } +# Check if token has changed and refresh if needed +_check_and_refresh_token() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + return 0 # Skip if token not available + fi + + local config_file="$HOME/.claude/settings.json" + local token_hash_file="$HOME/.claude/.token_hash" + + # Calculate current token hash + local current_hash + current_hash=$(echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 2>/dev/null || echo "") + + if [ -z "$current_hash" ]; then + return 0 # Skip if hash calculation failed + fi + + # Check if token has changed + if [ -f "$token_hash_file" ]; then + local stored_hash + stored_hash=$(cat "$token_hash_file" 2>/dev/null || echo "") + if [ "$current_hash" = "$stored_hash" ]; then + return 0 # Token unchanged, no refresh needed + fi + fi + + # Token changed or first time - refresh config + mkdir -p "$HOME/.claude" + if _generate_claude_config >/dev/null 2>&1; then + # Only show message if in interactive shell (not cron) + if [ -t 0 ]; then + echo "✓ Claude Code token refreshed automatically" + fi + return 0 + fi + + return 1 +} + # Auto-generate Claude settings from environment on first login # NOTE: settings.json acts as a FALLBACK - env vars (set above) are the primary method. # This is only generated if the file doesn't exist, to provide authentication when @@ -134,6 +178,12 @@ if [ ! -f "$HOME/.claude/settings.json" ] && [ -n "$DATABRICKS_TOKEN" ] && [ -n fi fi +# Auto-refresh token on shell login if it has changed +# This ensures settings.json stays in sync with DATABRICKS_TOKEN +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + _check_and_refresh_token +fi + # Auto-enable Claude tracing on login (if not already enabled) # This ensures tracing is always active and saves to the shared workspace path if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ] && command -v mlflow >/dev/null 2>&1; then @@ -174,10 +224,120 @@ claude-refresh-token() { fi mkdir -p "$HOME/.claude" - _generate_claude_config - echo "✓ Claude Code settings updated with:" - echo " DATABRICKS_HOST: $DATABRICKS_HOST" - echo " DATABRICKS_TOKEN: \${DATABRICKS_TOKEN:0:20}..." + if _generate_claude_config; then + echo "✓ Claude Code settings updated with:" + echo " DATABRICKS_HOST: $DATABRICKS_HOST" + echo " DATABRICKS_TOKEN: ${DATABRICKS_TOKEN:0:20}..." + else + echo "⚠ Failed to update Claude settings" + return 1 + fi +} + +# Setup cron job for periodic token refresh (runs hourly) +claude-setup-token-refresh() { + local cron_cmd="[ -n \"\$DATABRICKS_TOKEN\" ] && [ -n \"\$DATABRICKS_HOST\" ] && source \"\$HOME/.bashrc\" && _check_and_refresh_token >/dev/null 2>&1" + local cron_job="0 * * * * $cron_cmd" + local cron_file="$HOME/.claude/token-refresh-cron" + + # Create cron wrapper script + mkdir -p "$HOME/.claude" + cat > "$cron_file" <<'CRON_SCRIPT' +#!/bin/bash +# Auto-generated cron script for Claude token refresh +# This script is called by cron to refresh the Claude token periodically + +# Source bashrc to get functions +if [ -f "$HOME/.bashrc" ]; then + source "$HOME/.bashrc" >/dev/null 2>&1 +fi + +# Check and refresh token if needed +_check_and_refresh_token +CRON_SCRIPT + chmod +x "$cron_file" + + # Check if cron job already exists + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + echo "✓ Token refresh cron job already configured" + return 0 + fi + + # Add cron job + (crontab -l 2>/dev/null; echo "0 * * * * $cron_file") | crontab - + if [ $? -eq 0 ]; then + echo "✓ Token refresh cron job configured (runs hourly)" + echo " To remove: crontab -e" + else + echo "⚠ Failed to setup cron job (may require cron service)" + return 1 + fi +} + +# Remove token refresh cron job +claude-remove-token-refresh() { + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + crontab -l 2>/dev/null | grep -v "token-refresh-cron" | crontab - + echo "✓ Token refresh cron job removed" + else + echo "ℹ No token refresh cron job found" + fi +} + +# Check token freshness status +claude-token-status() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + echo "⚠ DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + return 1 + fi + + local config_file="$HOME/.claude/settings.json" + local token_hash_file="$HOME/.claude/.token_hash" + + echo "=== Claude Token Status ===" + echo "" + + # Check if config file exists + if [ -f "$config_file" ]; then + echo "✓ Settings file: $config_file" + local file_age + file_age=$(stat -c %Y "$config_file" 2>/dev/null || stat -f %m "$config_file" 2>/dev/null || echo "0") + local current_time + current_time=$(date +%s) + local age_hours + age_hours=$(( (current_time - file_age) / 3600 )) + echo " Last updated: ${age_hours} hour(s) ago" + else + echo "✗ Settings file: missing" + fi + + echo "" + + # Check token hash + if [ -f "$token_hash_file" ]; then + local current_hash + current_hash=$(echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 2>/dev/null || echo "") + local stored_hash + stored_hash=$(cat "$token_hash_file" 2>/dev/null || echo "") + if [ "$current_hash" = "$stored_hash" ] && [ -n "$current_hash" ]; then + echo "✓ Token: matches stored hash (up to date)" + else + echo "⚠ Token: differs from stored hash (needs refresh)" + echo " Run: claude-refresh-token" + fi + else + echo "ℹ Token hash: not stored (will be created on next refresh)" + fi + + echo "" + + # Check cron job + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + echo "✓ Auto-refresh: enabled (hourly cron job)" + else + echo "ℹ Auto-refresh: disabled" + echo " Enable with: claude-setup-token-refresh" + fi } claude-tracing-enable() { @@ -283,6 +443,18 @@ check-claude() { fi echo "" + # VS Code/Cursor Remote SSH info + echo "VS Code/Cursor Remote SSH:" + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo " ✓ Python virtual environment: $venv_path" + echo " Run 'claude-vscode-setup' for setup instructions" + else + echo " ℹ Run 'claude-vscode-setup' for Remote SSH setup guide" + fi + echo "" + echo "Run 'source ~/.bashrc' if commands are still not found" } @@ -298,6 +470,181 @@ claude-debug() { echo "Claude config directory:" ls -la "$HOME/.claude/" 2>/dev/null || echo " Directory doesn't exist" } + +# VS Code/Cursor Remote SSH helpers +claude-vscode-env() { + # Show the Databricks virtual environment path for VS Code/Cursor + if [ -n "$DATABRICKS_VIRTUAL_ENV" ]; then + echo "$DATABRICKS_VIRTUAL_ENV" + else + # Try to find pythonEnv-* directories + local python_envs + python_envs=$(find /databricks/python* -maxdepth 1 -type d -name "pythonEnv-*" 2>/dev/null | head -1) + if [ -n "$python_envs" ]; then + echo "$python_envs" + else + echo "⚠ DATABRICKS_VIRTUAL_ENV not set and pythonEnv-* not found" + echo " Try: echo \$DATABRICKS_VIRTUAL_ENV" + return 1 + fi + fi +} + +claude-vscode-setup() { + echo "=== VS Code/Cursor Remote SSH Setup Guide ===" + echo "" + echo "1. Install Remote SSH Extension" + echo " - VS Code: Install 'Remote - SSH' extension" + echo " - Cursor: Built-in Remote SSH extension (already included)" + echo "" + echo "2. Configure Default Extensions" + echo " Open Command Palette (Cmd+Shift+P / Ctrl+Shift+P):" + echo " → Remote-SSH: Settings" + echo "" + echo " Or edit settings.json and add:" + echo "" + cat <<'VSCODE_SETTINGS' + "remote.SSH.defaultExtensions": [ + "ms-Python.python", + "ms-toolsai.jupyter" + ] +VSCODE_SETTINGS + echo "" + echo "3. Connect to Cluster" + echo " - Command Palette → Remote-SSH: Connect to Host" + echo " - Enter your cluster SSH connection details" + echo "" + echo "4. Select Python Interpreter" + echo " After connecting, run this command to get the Python path:" + echo "" + echo " $ claude-vscode-env" + echo "" + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo " Current virtual environment:" + echo " $venv_path" + echo "" + echo " Then in VS Code/Cursor:" + echo " - Command Palette → Python: Select Interpreter" + echo " - Paste the path above or browse to it" + else + echo " Run 'echo \$DATABRICKS_VIRTUAL_ENV' to find the path" + fi + echo "" + echo "5. Important Notes" + echo " • IPYNB notebooks and *.py Databricks notebooks have access to" + echo " Databricks globals (dbutils, spark, etc.)" + echo " • Regular Python *.py files do NOT have access to Databricks globals" + echo " • Always select the pythonEnv-xxx interpreter for full Databricks" + echo " Runtime library access" + echo "" + echo "6. Verify Setup" + echo " Run: claude-vscode-check" +} + +claude-vscode-check() { + echo "=== VS Code/Cursor Remote SSH Setup Check ===" + echo "" + + # Check for virtual environment + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "✓ Python Virtual Environment:" + echo " $venv_path" + if [ -d "$venv_path/bin" ]; then + echo " ✓ Virtual environment directory exists" + if [ -f "$venv_path/bin/python" ]; then + echo " ✓ Python executable found" + echo " Python version: $($venv_path/bin/python --version 2>&1 || echo 'unknown')" + else + echo " ⚠ Python executable not found" + fi + else + echo " ⚠ Virtual environment directory not found" + fi + else + echo "✗ Python Virtual Environment: Not found" + echo " Run: echo \$DATABRICKS_VIRTUAL_ENV" + fi + echo "" + + # Check for Python + if command -v python3 >/dev/null 2>&1; then + echo "✓ Python3 available: $(which python3)" + echo " Version: $(python3 --version 2>&1)" + else + echo "✗ Python3 not found in PATH" + fi + echo "" + + # Check for Databricks runtime libraries + echo "Databricks Runtime Libraries:" + python3 <<'PYTHON_CHECK' +import sys +libraries = ['pyspark', 'pandas', 'numpy', 'mlflow', 'databricks'] +found = [] +missing = [] + +for lib in libraries: + try: + __import__(lib) + found.append(lib) + except ImportError: + missing.append(lib) + +if found: + print(f" ✓ Available: {', '.join(found)}") +if missing: + print(f" ⚠ Missing: {', '.join(missing)}") + +# Check for Databricks globals (only available in notebooks) +try: + import dbutils + print(" ✓ dbutils available (notebook context)") +except: + print(" ℹ dbutils not available (normal for .py files)") +PYTHON_CHECK + + echo "" + echo "VS Code/Cursor Configuration:" + echo " Run 'claude-vscode-setup' for setup instructions" + echo " Run 'claude-vscode-env' to get Python interpreter path" +} + +claude-vscode-config() { + # Generate VS Code settings.json snippet + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + + echo "=== VS Code/Cursor settings.json Configuration ===" + echo "" + echo "Add this to your VS Code/Cursor settings.json:" + echo "" + echo "{" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "," + echo " \"python.defaultInterpreterPath\": \"$venv_path/bin/python\"" + fi + echo "}" + echo "" + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "Python interpreter path:" + echo " $venv_path/bin/python" + echo "" + echo "To set this in VS Code/Cursor:" + echo " 1. Command Palette → Python: Select Interpreter" + echo " 2. Enter interpreter path: $venv_path/bin/python" + else + echo "To find Python interpreter path, run:" + echo " claude-vscode-env" + fi +} ### CLAUDE_CODE_HELPERS_END ### EOF @@ -349,12 +696,33 @@ main() { log " 1. Run: source ~/.bashrc" log " 2. Verify: check-claude" log " 3. Use: claude command" + log "" + # Setup automatic token refresh (optional - user can enable manually) + log "Setting up automatic token refresh..." + if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + # Setup cron job for periodic refresh + if command -v crontab >/dev/null 2>&1; then + # Source bashrc temporarily to get the function + source "$HOME/.bashrc" >/dev/null 2>&1 || true + claude-setup-token-refresh >/dev/null 2>&1 || log "⚠ Cron setup skipped (may require manual setup)" + else + log "⚠ Cron not available - token refresh will only happen on login" + fi + fi + log "" log "Helper commands:" log " - check-claude: Verify installation status" log " - claude-debug: Show Claude CLI configuration details" log " - claude-refresh-token: Regenerate Claude settings" + log " - claude-token-status: Check token freshness and auto-refresh status" + log " - claude-setup-token-refresh: Enable hourly automatic token refresh" + log " - claude-remove-token-refresh: Disable automatic token refresh" log " - claude-tracing-enable/disable/status: Manage MLflow tracing" + log " - claude-vscode-setup: Show VS Code/Cursor Remote SSH setup guide" + log " - claude-vscode-env: Get Python virtual environment path" + log " - claude-vscode-check: Verify VS Code/Cursor setup" + log " - claude-vscode-config: Generate VS Code settings.json snippet" return 0 } diff --git a/modules/adb-coding-assistants-cluster/scripts/vscode-setup.sh b/modules/adb-coding-assistants-cluster/scripts/vscode-setup.sh new file mode 100755 index 00000000..89b21ada --- /dev/null +++ b/modules/adb-coding-assistants-cluster/scripts/vscode-setup.sh @@ -0,0 +1,248 @@ +#!/bin/bash +# +# VS Code/Cursor Remote SSH Setup Helper for Databricks Clusters +# This script helps configure VS Code or Cursor for remote development on Databricks clusters +# + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +log_info() { echo -e "${BLUE}ℹ${NC} $1"; } +log_success() { echo -e "${GREEN}✓${NC} $1"; } +log_warning() { echo -e "${YELLOW}⚠${NC} $1"; } +log_error() { echo -e "${RED}✗${NC} $1"; } + +# Find Databricks Python virtual environment +find_python_env() { + if [ -n "${DATABRICKS_VIRTUAL_ENV:-}" ]; then + echo "$DATABRICKS_VIRTUAL_ENV" + return 0 + fi + + # Try to find pythonEnv-* directories + local python_envs + python_envs=$(find /databricks/python* -maxdepth 1 -type d -name "pythonEnv-*" 2>/dev/null | head -1) + if [ -n "$python_envs" ]; then + echo "$python_envs" + return 0 + fi + + return 1 +} + +# Show setup instructions +show_setup_guide() { + echo "==========================================" + echo "VS Code/Cursor Remote SSH Setup Guide" + echo "==========================================" + echo "" + + echo "1. Install Remote SSH Extension" + echo " • VS Code: Install 'Remote - SSH' extension from marketplace" + echo " • Cursor: Built-in Remote SSH extension (already included)" + echo "" + + echo "2. Configure Default Extensions" + echo " Open Command Palette (Cmd+Shift+P / Ctrl+Shift+P):" + echo " → Type: Remote-SSH: Settings" + echo "" + echo " Or manually edit settings.json:" + echo "" + echo " {" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + echo " }" + echo "" + + echo "3. Connect to Cluster" + echo " • Command Palette → Remote-SSH: Connect to Host" + echo " • Enter your cluster SSH connection details" + echo " • Format: user@hostname or use SSH config entry" + echo "" + + echo "4. Select Python Interpreter" + local venv_path + if venv_path=$(find_python_env 2>/dev/null); then + echo " ✓ Found Python virtual environment:" + echo " $venv_path" + echo "" + echo " In VS Code/Cursor:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Enter interpreter path:" + echo " $venv_path/bin/python" + echo "" + echo " Or copy this path:" + echo " $venv_path/bin/python" + else + echo " ⚠ Could not auto-detect Python virtual environment" + echo " Run this command to find it:" + echo " echo \$DATABRICKS_VIRTUAL_ENV" + echo "" + echo " Then in VS Code/Cursor:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Paste the path from above" + fi + echo "" + + echo "5. Important Notes" + echo " • IPYNB notebooks and *.py Databricks notebooks have access to" + echo " Databricks globals (dbutils, spark, etc.)" + echo " • Regular Python *.py files do NOT have access to Databricks globals" + echo " • Always select the pythonEnv-xxx interpreter for full Databricks" + echo " Runtime library access (pyspark, pandas, numpy, mlflow, etc.)" + echo "" + + echo "6. Verify Setup" + echo " After connecting, verify Python interpreter:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Should show: pythonEnv-xxx/bin/python" + echo "" + echo " Test in a Python file:" + echo " import pyspark" + echo " import pandas" + echo " print('Setup successful!')" +} + +# Generate VS Code settings.json snippet +generate_settings() { + local venv_path + venv_path=$(find_python_env 2>/dev/null || echo "") + + echo "{" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + if [ -n "$venv_path" ]; then + echo "," + echo " \"python.defaultInterpreterPath\": \"$venv_path/bin/python\"" + fi + echo "}" +} + +# Check current setup +check_setup() { + echo "==========================================" + echo "VS Code/Cursor Setup Check" + echo "==========================================" + echo "" + + # Check for virtual environment + local venv_path + if venv_path=$(find_python_env 2>/dev/null); then + log_success "Python Virtual Environment found:" + echo " $venv_path" + + if [ -d "$venv_path/bin" ]; then + log_success "Virtual environment directory exists" + if [ -f "$venv_path/bin/python" ]; then + log_success "Python executable found" + echo " Python version: $($venv_path/bin/python --version 2>&1 || echo 'unknown')" + else + log_warning "Python executable not found" + fi + else + log_warning "Virtual environment directory not found" + fi + else + log_error "Python Virtual Environment not found" + echo " Run: echo \$DATABRICKS_VIRTUAL_ENV" + fi + echo "" + + # Check for Python + if command -v python3 >/dev/null 2>&1; then + log_success "Python3 available: $(which python3)" + echo " Version: $(python3 --version 2>&1)" + else + log_error "Python3 not found in PATH" + fi + echo "" + + # Check for Databricks runtime libraries + echo "Databricks Runtime Libraries:" + python3 <<'PYTHON_CHECK' +import sys +libraries = ['pyspark', 'pandas', 'numpy', 'mlflow'] +found = [] +missing = [] + +for lib in libraries: + try: + __import__(lib) + found.append(lib) + except ImportError: + missing.append(lib) + +if found: + print(f" ✓ Available: {', '.join(found)}") +if missing: + print(f" ⚠ Missing: {', '.join(missing)}") + +# Check for Databricks globals (only available in notebooks) +try: + import dbutils + print(" ✓ dbutils available (notebook context)") +except: + print(" ℹ dbutils not available (normal for .py files)") +PYTHON_CHECK + + echo "" + echo "Next steps:" + echo " • Run this script with --guide to see setup instructions" + echo " • Run this script with --settings to generate settings.json" +} + +# Main +main() { + case "${1:-}" in + --guide|-g) + show_setup_guide + ;; + --settings|-s) + generate_settings + ;; + --check|-c) + check_setup + ;; + --env|-e) + find_python_env || { + log_error "Could not find Python virtual environment" + echo "Try: echo \$DATABRICKS_VIRTUAL_ENV" + exit 1 + } + ;; + --help|-h|"") + echo "VS Code/Cursor Remote SSH Setup Helper" + echo "" + echo "Usage: $0 [OPTION]" + echo "" + echo "Options:" + echo " --guide, -g Show complete setup guide" + echo " --settings, -s Generate VS Code settings.json snippet" + echo " --check, -c Check current setup status" + echo " --env, -e Show Python virtual environment path" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 --guide # Show setup instructions" + echo " $0 --env # Get Python interpreter path" + echo " $0 --check # Verify setup" + ;; + *) + log_error "Unknown option: $1" + echo "Run '$0 --help' for usage information" + exit 1 + ;; + esac +} + +main "$@" From 0bf5627b52be6422fed1d87e3737d8e03468b951 Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Mon, 26 Jan 2026 22:09:33 +1100 Subject: [PATCH 07/14] fix: Resolve Azure provider subscription ID error for profile auth Add external data source to fetch Azure subscription ID from Azure CLI or environment variable when using profile-based Databricks authentication. This fixes the error where Azure provider could not determine subscription ID when databricks_resource_id is not provided. - Add external provider to required_providers - Add data source to get subscription ID from Azure CLI or ARM_SUBSCRIPTION_ID - Update provider configuration to use subscription ID from multiple sources --- .../providers.tf | 70 ++++++++++++++----- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/examples/adb-coding-assistants-cluster/providers.tf b/examples/adb-coding-assistants-cluster/providers.tf index 906fe89d..26c208e7 100644 --- a/examples/adb-coding-assistants-cluster/providers.tf +++ b/examples/adb-coding-assistants-cluster/providers.tf @@ -8,46 +8,84 @@ terraform { source = "databricks/databricks" version = ">=1.81.1" } + external = { + source = "hashicorp/external" + version = ">=2.3.0" + } } } -# Data source to get current Azure client configuration -data "azurerm_client_config" "current" {} +# Determine authentication approach based on variables provided +locals { + # Use profile-based auth if profile is specified + use_profile_auth = var.databricks_profile != null + + # For Azure resource ID approach + resource_regex = var.databricks_resource_id != null ? "(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft.Databricks/workspaces/(.+)" : "" + subscription_id_from_resource = var.databricks_resource_id != null ? regex(local.resource_regex, var.databricks_resource_id)[0] : null + resource_group = var.databricks_resource_id != null ? regex(local.resource_regex, var.databricks_resource_id)[1] : null + databricks_workspace_name = var.databricks_resource_id != null ? regex(local.resource_regex, var.databricks_resource_id)[2] : null +} + +# Get Azure subscription ID from Azure CLI or environment variable when not provided via resource ID +# This is needed for the Azure provider even when using profile-based Databricks auth +data "external" "azure_subscription" { + count = local.subscription_id_from_resource == null ? 1 : 0 + program = ["bash", "-c", "SUBSCRIPTION_ID=$(az account show --query id -o tsv 2>/dev/null || echo $${ARM_SUBSCRIPTION_ID:-}); echo \"{\\\"id\\\":\\\"$${SUBSCRIPTION_ID:-}\\\"}\""] +} -# Extract subscription ID, resource group, and workspace name from resource ID locals { - resource_regex = "(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft.Databricks/workspaces/(.+)" - subscription_id = regex(local.resource_regex, var.databricks_resource_id)[0] - resource_group = regex(local.resource_regex, var.databricks_resource_id)[1] - databricks_workspace_name = regex(local.resource_regex, var.databricks_resource_id)[2] + # Use subscription ID from resource ID, or from Azure CLI/environment, or null (provider will try to auto-detect) + subscription_id = coalesce( + local.subscription_id_from_resource, + try(data.external.azure_subscription[0].result.id != "" ? data.external.azure_subscription[0].result.id : null, null) + ) +} + +# Data source to get current Azure client configuration (only for Azure resource ID approach) +data "azurerm_client_config" "current" { + count = local.use_profile_auth ? 0 : 1 } -# Data source to get the resource group +# Data source to get the resource group (only for Azure resource ID approach) data "azurerm_resource_group" "this" { - name = local.resource_group + count = local.use_profile_auth ? 0 : 1 + name = local.resource_group } # Configure the Azure Provider +# When using profile-based auth, subscription_id is not needed (provider will auto-detect if Azure CLI is configured) +# When using Azure resource ID approach, subscription_id is extracted from the resource ID provider "azurerm" { subscription_id = local.subscription_id features {} + skip_provider_registration = local.use_profile_auth + + # Allow provider to work without explicit subscription_id when using profile auth + # It will attempt to auto-detect from Azure CLI or environment variables } -# Data source to get the Databricks workspace +# Data source to get the Databricks workspace (only for Azure resource ID approach) data "azurerm_databricks_workspace" "this" { + count = local.use_profile_auth ? 0 : 1 name = local.databricks_workspace_name resource_group_name = local.resource_group } # Configure the Databricks Provider -# Authentication uses Databricks unified authentication: -# 1. Environment variables (DATABRICKS_HOST, DATABRICKS_TOKEN) - Recommended for CI/CD -# 2. Azure CLI authentication (az login) - Recommended for local development -# 3. Configuration profile (~/.databrickscfg) - Alternative for local development +# Two authentication approaches supported: +# +# 1. Profile-based (Recommended - Simple and cloud-agnostic): +# Set databricks_profile variable to your ~/.databrickscfg profile name +# Example: databricks_profile = "dok" +# +# 2. Azure resource ID (Azure-specific): +# Set databricks_resource_id to your Azure Databricks workspace resource ID +# Requires Azure CLI authentication (az login) # # See: https://docs.databricks.com/dev-tools/auth/unified-auth.html provider "databricks" { - host = data.azurerm_databricks_workspace.this.workspace_url - # No explicit authentication configured - uses unified authentication + profile = var.databricks_profile + host = local.use_profile_auth ? null : data.azurerm_databricks_workspace.this[0].workspace_url } From 72316cb9aeb891198e5f2f6146279a0ca152ff54 Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Mon, 26 Jan 2026 22:09:34 +1100 Subject: [PATCH 08/14] refactor: Remove auto-execution from Claude init script Remove automatic MLflow tracing enable and cron setup that could cause script failures. Keep all helper functions available for manual use. - Remove auto-enable MLflow tracing on login - Remove automatic cron job setup in main() - Keep all helper functions (claude-tracing-enable, claude-setup-token-refresh, etc.) - Users can manually enable features if desired --- .../scripts/install-claude.sh | 47 +------------------ 1 file changed, 1 insertion(+), 46 deletions(-) mode change 100644 => 100755 modules/adb-coding-assistants-cluster/scripts/install-claude.sh diff --git a/modules/adb-coding-assistants-cluster/scripts/install-claude.sh b/modules/adb-coding-assistants-cluster/scripts/install-claude.sh old mode 100644 new mode 100755 index 605f0292..c39d9cfa --- a/modules/adb-coding-assistants-cluster/scripts/install-claude.sh +++ b/modules/adb-coding-assistants-cluster/scripts/install-claude.sh @@ -184,37 +184,6 @@ if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then _check_and_refresh_token fi -# Auto-enable Claude tracing on login (if not already enabled) -# This ensures tracing is always active and saves to the shared workspace path -if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ] && command -v mlflow >/dev/null 2>&1; then - # Check if tracing is already enabled (non-zero exit means not enabled) - if ! mlflow autolog claude --status >/dev/null 2>&1; then - # Create experiment if it doesn't exist - python3 </dev/null 2>&1) && \ - echo "✓ Claude Code MLflow tracing auto-enabled in $WORKSPACE_DIR (experiment: EXP_PH)" - fi -fi - # Regenerate Claude settings from current environment claude-refresh-token() { if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then @@ -696,27 +665,13 @@ main() { log " 1. Run: source ~/.bashrc" log " 2. Verify: check-claude" log " 3. Use: claude command" - log "" - # Setup automatic token refresh (optional - user can enable manually) - log "Setting up automatic token refresh..." - if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then - # Setup cron job for periodic refresh - if command -v crontab >/dev/null 2>&1; then - # Source bashrc temporarily to get the function - source "$HOME/.bashrc" >/dev/null 2>&1 || true - claude-setup-token-refresh >/dev/null 2>&1 || log "⚠ Cron setup skipped (may require manual setup)" - else - log "⚠ Cron not available - token refresh will only happen on login" - fi - fi - log "" log "Helper commands:" log " - check-claude: Verify installation status" log " - claude-debug: Show Claude CLI configuration details" log " - claude-refresh-token: Regenerate Claude settings" log " - claude-token-status: Check token freshness and auto-refresh status" - log " - claude-setup-token-refresh: Enable hourly automatic token refresh" + log " - claude-setup-token-refresh: Enable hourly automatic token refresh (optional)" log " - claude-remove-token-refresh: Disable automatic token refresh" log " - claude-tracing-enable/disable/status: Manage MLflow tracing" log " - claude-vscode-setup: Show VS Code/Cursor Remote SSH setup guide" From 04e0059a0e0ad9dd9e0678f519cadeb383ff3418 Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Mon, 26 Jan 2026 22:09:36 +1100 Subject: [PATCH 09/14] docs: Add Unity Catalog storage credential prerequisite Document that Unity Catalog metastore must have a root storage credential configured before volumes can be created. This is required for the module to work properly. --- .../adb-coding-assistants-cluster/README.md | 84 +++++++++++++++++++ .../adb-coding-assistants-cluster/README.md | 61 ++++++++++++-- 2 files changed, 139 insertions(+), 6 deletions(-) diff --git a/examples/adb-coding-assistants-cluster/README.md b/examples/adb-coding-assistants-cluster/README.md index bc6a17a3..6dfaa8d7 100644 --- a/examples/adb-coding-assistants-cluster/README.md +++ b/examples/adb-coding-assistants-cluster/README.md @@ -32,10 +32,13 @@ This template provides an example deployment of a Databricks cluster pre-configu - Databricks workspace with Unity Catalog enabled - Unity Catalog with an existing catalog and schema +- **Unity Catalog metastore must have a root storage credential configured** (required for volumes) - Permission to create clusters - (For Azure) Authenticated via `az login` or environment variables - Databricks Runtime 14.3 LTS or higher recommended +> **Note**: If you encounter an error about missing root storage credential, you need to configure the metastore's root storage credential first. See [Databricks documentation](https://docs.databricks.com/api-explorer/workspace/metastores/update) for details. + ## Post-Deployment After the cluster starts, you can connect via SSH to use Claude Code and other development tools. @@ -64,6 +67,11 @@ This creates an entry in your `~/.ssh/config` file. 5. Select **Linux** as the platform. 6. Once connected, open your persistent workspace folder: `/Workspace/Users//`. +> **Important: Work Storage Location** +> ⚠️ **DO NOT use Databricks Repos (`/Repos/...`) for active development work.** Repos folders can be unreliable for persistent storage and may lose uncommitted changes during cluster restarts or sync operations. +> +> ✅ **Use `/Workspace/Users//` instead.** This location provides reliable persistent storage. You can use regular git commands to manage version control (see "Using Git in /Workspace" section below). + ### 3. Launch Claude Code Open the terminal in your remote VSCode/Cursor session and run: @@ -126,17 +134,93 @@ tmux attach -t claude-session This allows you to leave long-running tasks (like "Build a data pipeline") executing on the cluster while you are offline. +### 7. Using Git in /Workspace + +Since `/Workspace` doesn't have native Repos integration, use standard git commands: + +```bash +# Navigate to your workspace directory +cd /Workspace/Users// + +# Option 1: Clone an existing repository +git clone https://github.com/your-org/your-repo.git +cd your-repo + +# Option 2: Initialize a new repository +mkdir my-project && cd my-project +git init +git remote add origin https://github.com/your-org/your-repo.git + +# Configure git (first time only) +git config user.name "Your Name" +git config user.email "your.email@company.com" + +# Regular git workflow +git add . +git commit -m "Your commit message" +git push origin main +``` + +**Git Authentication Options:** + +1. **Personal Access Token (PAT)** - Recommended: + ```bash + # GitHub: Create at https://github.com/settings/tokens + # Use token as password when prompted + git clone https://github.com/your-org/repo.git + ``` + +2. **SSH Keys**: + ```bash + # Generate SSH key on the cluster + ssh-keygen -t ed25519 -C "your.email@company.com" + + # Add to GitHub: Copy output and add at https://github.com/settings/keys + cat ~/.ssh/id_ed25519.pub + + # Clone using SSH + git clone git@github.com:your-org/repo.git + ``` + +3. **Git Credential Manager**: + ```bash + # Store credentials to avoid repeated prompts + git config --global credential.helper store + ``` + ## Helper Commands +### Claude CLI Commands + | Command | Purpose | |---------|---------| | `check-claude` | Verify Claude CLI installation and configuration | | `claude-debug` | Show detailed Claude configuration | | `claude-refresh-token` | Regenerate Claude settings from environment | +| `claude-token-status` | Check token freshness and auto-refresh status | | `claude-tracing-enable` | Enable MLflow tracing for Claude sessions | | `claude-tracing-status` | Check tracing status | | `claude-tracing-disable` | Disable tracing | +### Git Workspace Commands + +| Command | Purpose | +|---------|---------| +| `git-workspace-init` | Interactive setup for git in /Workspace (clone or init) | +| `git-workspace-check` | Verify location and check for uncommitted/unpushed changes | +| `git-workspace-setup-auth` | Configure git authentication (PAT, SSH, or credential helper) | + +These helpers warn you if working in `/Repos` and ensure your work is backed up in git. + +### VS Code/Cursor Remote Commands + +| Command | Purpose | +|---------|---------| +| `claude-vscode-setup` | Show Remote SSH setup instructions | +| `claude-vscode-env` | Get Python interpreter path for IDE | +| `claude-vscode-check` | Verify Remote SSH configuration | +| `claude-vscode-config` | Generate settings.json snippet | + ## Offline Installation For air-gapped or restricted network environments, use the separate offline module: [`adb-coding-assistants-cluster-offline`](../../modules/adb-coding-assistants-cluster-offline/README.md). See the [Offline Installation Guide](../../modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md) for detailed instructions. diff --git a/modules/adb-coding-assistants-cluster/README.md b/modules/adb-coding-assistants-cluster/README.md index 71df1f5c..8838f877 100644 --- a/modules/adb-coding-assistants-cluster/README.md +++ b/modules/adb-coding-assistants-cluster/README.md @@ -56,6 +56,9 @@ This module can be used to deploy the following: - Databricks Runtime 13.3 LTS or higher (recommended for Unity Catalog volumes) - Databricks Terraform provider >= 1.40.0 - Unity Catalog with an existing catalog and schema +- **Unity Catalog metastore must have a root storage credential configured** (required for volumes) + +> **Note**: If you encounter an error about missing root storage credential, you need to configure the metastore's root storage credential first. See [Databricks documentation](https://docs.databricks.com/api-explorer/workspace/metastores/update) for details. ## Usage @@ -233,7 +236,7 @@ After the cluster starts, users can: ```bash # Check installation status -check-coding-assistants +check-claude # Debug Claude configuration claude-debug @@ -241,9 +244,6 @@ claude-debug # Use Claude Code claude "Analyze the customer churn data" -# Use OpenCode -opencode "Generate unit tests for my functions" - # Enable MLflow tracing claude-tracing-enable @@ -251,20 +251,69 @@ claude-tracing-enable claude-tracing-status ``` +### Persistent Work Storage + +**IMPORTANT: Do not use Databricks Repos (`/Repos/...`) for active development work.** + +Databricks Repos folders can be unreliable for persistent storage and may lose uncommitted changes during cluster restarts or sync operations. Instead: + +✅ **Use `/Workspace/Users//` for all development work** + +This location provides reliable persistent storage across cluster restarts. Use the provided git helpers to manage version control: + +```bash +# Navigate to your workspace +cd /Workspace/Users/$(whoami)/ + +# Set up git (interactive helper) +git-workspace-init + +# Check git status and location +git-workspace-check + +# Configure git authentication +git-workspace-setup-auth +``` + +The git helpers will: +- Warn if you're working in `/Repos` (unreliable location) +- Help you clone existing repos or initialize new ones +- Check for uncommitted or unpushed changes +- Guide you through authentication setup (PAT, SSH, or credential helper) + ### Helper Commands The init script installs these helper commands in `~/.bashrc`: +#### Claude CLI Commands + | Command | Purpose | |---------|---------| -| `check-coding-assistants` | Verify installation and configuration | +| `check-claude` | Verify installation and configuration | | `claude-debug` | Show detailed Claude CLI configuration | | `claude-refresh-token` | Regenerate Claude settings | -| `opencode-refresh-config` | Regenerate OpenCode config | +| `claude-token-status` | Check token freshness and auto-refresh status | | `claude-tracing-enable` | Enable MLflow tracing | | `claude-tracing-status` | Check tracing status | | `claude-tracing-disable` | Disable MLflow tracing | +#### Git Workspace Commands + +| Command | Purpose | +|---------|---------| +| `git-workspace-init` | Interactive git setup in /Workspace (clone or init) | +| `git-workspace-check` | Check location and uncommitted/unpushed changes | +| `git-workspace-setup-auth` | Configure git authentication (PAT/SSH/credential helper) | + +#### VS Code/Cursor Remote Commands + +| Command | Purpose | +|---------|---------| +| `claude-vscode-setup` | Show Remote SSH setup guide | +| `claude-vscode-env` | Get Python interpreter path | +| `claude-vscode-check` | Verify Remote SSH configuration | +| `claude-vscode-config` | Generate settings.json snippet | + ## Cluster Access Modes ### Single-User Access Mode From 83d42f54aed6778e17616972620b0780b15deae8 Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Mon, 26 Jan 2026 22:09:44 +1100 Subject: [PATCH 10/14] feat: Add profile-based authentication option Add support for Databricks CLI profile-based authentication as an alternative to Azure resource ID. This provides a simpler, cloud-agnostic authentication method. - Add databricks_profile variable to example and module - Update variable descriptions to clarify authentication options - Update terraform.tfvars.example with both authentication methods - Add validation to ensure at least one auth method is provided --- .../terraform.tfvars.example | 34 ++++++++++++++----- .../variables.tf | 14 +++++++- .../variables.tf | 2 +- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/examples/adb-coding-assistants-cluster/terraform.tfvars.example b/examples/adb-coding-assistants-cluster/terraform.tfvars.example index 25c6de60..32296df2 100644 --- a/examples/adb-coding-assistants-cluster/terraform.tfvars.example +++ b/examples/adb-coding-assistants-cluster/terraform.tfvars.example @@ -1,12 +1,28 @@ # Example terraform.tfvars file for Claude Code CLI Cluster # Copy this to terraform.tfvars and customize for your environment -# Required variables -databricks_resource_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Databricks/workspaces/my-workspace" -cluster_name = "claude-coding-assistant" -catalog_name = "main" +#============================================================================= +# AUTHENTICATION - Choose ONE approach +#============================================================================= -# Optional variables with recommended defaults +# OPTION 1: Profile-based (Recommended - Simple and cloud-agnostic) +# Uses your ~/.databrickscfg profile +databricks_profile = "my-profile" # Replace with your profile name from ~/.databrickscfg + +# OPTION 2: Azure Resource ID (Azure-specific) +# Comment out databricks_profile above and uncomment below to use Azure resource ID +# databricks_resource_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Databricks/workspaces/my-workspace" + +#============================================================================= +# REQUIRED VARIABLES +#============================================================================= + +cluster_name = "claude-coding-assistant" +catalog_name = "main" + +#============================================================================= +# OPTIONAL VARIABLES +#============================================================================= # Unity Catalog configuration schema_name = "default" @@ -23,7 +39,7 @@ autotermination_minutes = 30 cluster_mode = "SINGLE_NODE" # Worker configuration (ignored if cluster_mode = "SINGLE_NODE") -num_workers = 0 # Set to null to enable autoscaling, or a specific number +num_workers = 0 # Set to null to enable autoscaling, or a specific number # Autoscaling configuration (only used if num_workers = null) min_workers = 1 @@ -43,8 +59,10 @@ tags = { CostCenter = "engineering" } -# Cloud-specific node types: -# +#============================================================================= +# CLOUD-SPECIFIC NODE TYPES REFERENCE +#============================================================================= + # Azure VM Types (Premium SSD): # Modern Dpdsv6-series (Cobalt 100 processor, Premium SSD + local NVMe): # - Standard_D4pds_v6 (4 cores, 16 GB RAM) - Cost-effective diff --git a/examples/adb-coding-assistants-cluster/variables.tf b/examples/adb-coding-assistants-cluster/variables.tf index b73fc922..4dc0f3ac 100644 --- a/examples/adb-coding-assistants-cluster/variables.tf +++ b/examples/adb-coding-assistants-cluster/variables.tf @@ -1,6 +1,18 @@ +variable "databricks_profile" { + description = "Databricks CLI profile name from ~/.databrickscfg (recommended for simple, cloud-agnostic authentication). If set, databricks_resource_id is ignored." + type = string + default = null +} + variable "databricks_resource_id" { - description = "The Azure resource ID for the Databricks workspace. Format: /subscriptions/{subscription-id}/resourceGroups/{resource-group}/providers/Microsoft.Databricks/workspaces/{workspace-name}" + description = "The Azure resource ID for the Databricks workspace (Azure-specific approach). Format: /subscriptions/{subscription-id}/resourceGroups/{resource-group}/providers/Microsoft.Databricks/workspaces/{workspace-name}. Only used if databricks_profile is not set." type = string + default = null + + validation { + condition = var.databricks_profile != null || var.databricks_resource_id != null + error_message = "Either databricks_profile or databricks_resource_id must be set. Recommended: use databricks_profile for simpler configuration." + } } variable "cluster_name" { diff --git a/modules/adb-coding-assistants-cluster/variables.tf b/modules/adb-coding-assistants-cluster/variables.tf index 61a40093..cb751f24 100644 --- a/modules/adb-coding-assistants-cluster/variables.tf +++ b/modules/adb-coding-assistants-cluster/variables.tf @@ -4,7 +4,7 @@ variable "cluster_name" { } variable "catalog_name" { - description = "Unity Catalog name for the volume" + description = "Unity Catalog catalog name for the volume. The metastore must have a root storage credential configured." type = string } From 32c0689e167a4f8c61bb0c97b89eb55df9ae8dce Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Tue, 3 Feb 2026 16:53:56 +1100 Subject: [PATCH 11/14] feat: Disable experimental betas in Claude Code CLI Add CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 environment variable to both init scripts (install-claude.sh and install-claude-minimal.sh) to disable experimental beta features when running Claude Code on Databricks clusters. The variable is exported in bashrc and set in the environment to ensure it persists across shell sessions and is available when the cluster starts. Co-authored-by: Cursor --- .../scripts/install-claude-minimal.sh | 66 +++++ .../scripts/install-claude.sh | 235 ++++++++++++------ 2 files changed, 221 insertions(+), 80 deletions(-) create mode 100755 modules/adb-coding-assistants-cluster/scripts/install-claude-minimal.sh diff --git a/modules/adb-coding-assistants-cluster/scripts/install-claude-minimal.sh b/modules/adb-coding-assistants-cluster/scripts/install-claude-minimal.sh new file mode 100755 index 00000000..5679d84e --- /dev/null +++ b/modules/adb-coding-assistants-cluster/scripts/install-claude-minimal.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Databricks Cluster Init Script - Claude Code CLI (Minimal Version) +# Installs Claude Code CLI with basic configuration only +# + +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +LOG_FILE="/tmp/init-script-claude.log" +log() { + echo "[$(date '+%H:%M:%S')] $1" | tee -a "$LOG_FILE" +} + +# Install system dependencies +log "Installing system dependencies..." +sudo apt-get update -qq -y >> "$LOG_FILE" 2>&1 +sudo apt-get install -y -qq curl git >> "$LOG_FILE" 2>&1 || log "Warning: Some packages failed to install" + +# Install Node.js 20.x +if ! command -v node >/dev/null 2>&1; then + log "Installing Node.js 20.x..." + curl -fsSL --max-time 300 --retry 3 https://deb.nodesource.com/setup_20.x | sudo -E bash - >> "$LOG_FILE" 2>&1 + sudo apt-get install -y -qq nodejs >> "$LOG_FILE" 2>&1 + log "Node.js installed: $(node --version)" +else + log "Node.js already installed: $(node --version)" +fi + +# Install Claude Code CLI +if ! command -v claude >/dev/null 2>&1; then + log "Installing Claude Code CLI..." + curl -fsSL https://claude.ai/install.sh | bash >> "$LOG_FILE" 2>&1 + log "Claude Code CLI installed" +else + log "Claude Code CLI already installed" +fi + +# Add basic configuration to bashrc +log "Configuring bashrc..." + +# Remove old Claude section if it exists +if [ -f "$HOME/.bashrc" ]; then + sed -i '/### CLAUDE_CODE_MINIMAL_START ###/,/### CLAUDE_CODE_MINIMAL_END ###/d' "$HOME/.bashrc" || true +fi + +# Add Claude to PATH and set environment variables +cat >> "$HOME/.bashrc" <<'BASHRC_EOF' + +### CLAUDE_CODE_MINIMAL_START ### +# Claude Code CLI - Minimal Setup +export PATH="$HOME/.claude/bin:$HOME/.local/bin:$PATH" + +# Set Anthropic environment variables for Claude CLI +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + export ANTHROPIC_AUTH_TOKEN="$DATABRICKS_TOKEN" + export ANTHROPIC_BASE_URL="${DATABRICKS_HOST}/serving-endpoints/anthropic" + export ANTHROPIC_MODEL="databricks-claude-sonnet-4-5" + export ANTHROPIC_CUSTOM_HEADERS="x-databricks-disable-beta-headers: true" + export CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 +fi +### CLAUDE_CODE_MINIMAL_END ### +BASHRC_EOF + +log "Configuration complete. Log file: $LOG_FILE" +log "After cluster starts, run: source ~/.bashrc" diff --git a/modules/adb-coding-assistants-cluster/scripts/install-claude.sh b/modules/adb-coding-assistants-cluster/scripts/install-claude.sh index c39d9cfa..52891fc2 100755 --- a/modules/adb-coding-assistants-cluster/scripts/install-claude.sh +++ b/modules/adb-coding-assistants-cluster/scripts/install-claude.sh @@ -16,16 +16,16 @@ cmd_exists() { command -v "$1" >/dev/null 2>&1; } # Install Claude Code CLI install_claude() { if cmd_exists claude; then - log "✓ Claude Code already installed" + log "[OK] Claude Code already installed" return 0 fi log "Installing Claude Code CLI..." if curl -fsSL https://claude.ai/install.sh | bash &>>$L; then - log "✓ Claude Code installation completed" + log "[OK] Claude Code installation completed" return 0 else - log "⚠ Claude Code installation failed (will be available after manual install)" + log "[WARN] Claude Code installation failed (will be available after manual install)" return 1 fi } @@ -33,7 +33,7 @@ install_claude() { # Install Node.js (required for Claude Code CLI) install_nodejs() { if cmd_exists node && cmd_exists npm; then - log "✓ Node.js already installed ($(node --version))" + log "[OK] Node.js already installed ($(node --version))" return 0 fi @@ -41,13 +41,13 @@ install_nodejs() { if curl -fsSL --max-time 300 --retry 3 https://deb.nodesource.com/setup_20.x | sudo -E bash - &>>$L; then if sudo apt-get update -qq -y &>>$L && sudo apt-get install -y -qq nodejs &>>$L; then if cmd_exists node && cmd_exists npm; then - log "✓ Node.js/npm installed successfully ($(node --version))" + log "[OK] Node.js/npm installed successfully ($(node --version))" return 0 fi fi fi - log "⚠ Node.js installation failed (Claude Code CLI will not work)" + log "[WARN] Node.js installation failed (Claude Code CLI will not work)" return 1 } @@ -93,6 +93,7 @@ if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then export ANTHROPIC_BASE_URL="${DATABRICKS_HOST}/serving-endpoints/anthropic" export ANTHROPIC_MODEL="databricks-claude-sonnet-4-5" export ANTHROPIC_CUSTOM_HEADERS="x-databricks-disable-beta-headers: true" + export CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 fi # Internal function to generate Claude settings (single source of truth) @@ -113,7 +114,7 @@ CLAUDE_CONFIG # Validate JSON if jq is available if command -v jq >/dev/null 2>&1; then if ! jq empty "$config_file" 2>/dev/null; then - echo "⚠ Claude settings JSON validation failed" >&2 + echo "[WARN] Claude settings JSON validation failed" >&2 return 1 fi fi @@ -157,7 +158,7 @@ _check_and_refresh_token() { if _generate_claude_config >/dev/null 2>&1; then # Only show message if in interactive shell (not cron) if [ -t 0 ]; then - echo "✓ Claude Code token refreshed automatically" + echo "[OK] Claude Code token refreshed automatically" fi return 0 fi @@ -172,9 +173,9 @@ _check_and_refresh_token() { if [ ! -f "$HOME/.claude/settings.json" ] && [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then mkdir -p "$HOME/.claude" if _generate_claude_config; then - echo "✓ Claude Code settings.json created (fallback - env vars take precedence)" + echo "[OK] Claude Code settings.json created (fallback - env vars take precedence)" else - echo "⚠ Failed to generate Claude settings (run claude-refresh-token to retry)" + echo "[WARN] Failed to generate Claude settings (run claude-refresh-token to retry)" fi fi @@ -187,18 +188,18 @@ fi # Regenerate Claude settings from current environment claude-refresh-token() { if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then - echo "⚠ DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + echo "[WARN] DATABRICKS_TOKEN and DATABRICKS_HOST must be set" echo " On Databricks clusters, these should be automatically available" return 1 fi mkdir -p "$HOME/.claude" if _generate_claude_config; then - echo "✓ Claude Code settings updated with:" + echo "[OK] Claude Code settings updated with:" echo " DATABRICKS_HOST: $DATABRICKS_HOST" echo " DATABRICKS_TOKEN: ${DATABRICKS_TOKEN:0:20}..." else - echo "⚠ Failed to update Claude settings" + echo "[WARN] Failed to update Claude settings" return 1 fi } @@ -228,17 +229,17 @@ CRON_SCRIPT # Check if cron job already exists if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then - echo "✓ Token refresh cron job already configured" + echo "[OK] Token refresh cron job already configured" return 0 fi # Add cron job (crontab -l 2>/dev/null; echo "0 * * * * $cron_file") | crontab - if [ $? -eq 0 ]; then - echo "✓ Token refresh cron job configured (runs hourly)" + echo "[OK] Token refresh cron job configured (runs hourly)" echo " To remove: crontab -e" else - echo "⚠ Failed to setup cron job (may require cron service)" + echo "[WARN] Failed to setup cron job (may require cron service)" return 1 fi } @@ -247,16 +248,16 @@ CRON_SCRIPT claude-remove-token-refresh() { if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then crontab -l 2>/dev/null | grep -v "token-refresh-cron" | crontab - - echo "✓ Token refresh cron job removed" + echo "[OK] Token refresh cron job removed" else - echo "ℹ No token refresh cron job found" + echo "[INFO] No token refresh cron job found" fi } # Check token freshness status claude-token-status() { if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then - echo "⚠ DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + echo "[WARN] DATABRICKS_TOKEN and DATABRICKS_HOST must be set" return 1 fi @@ -268,7 +269,7 @@ claude-token-status() { # Check if config file exists if [ -f "$config_file" ]; then - echo "✓ Settings file: $config_file" + echo "[OK] Settings file: $config_file" local file_age file_age=$(stat -c %Y "$config_file" 2>/dev/null || stat -f %m "$config_file" 2>/dev/null || echo "0") local current_time @@ -277,7 +278,7 @@ claude-token-status() { age_hours=$(( (current_time - file_age) / 3600 )) echo " Last updated: ${age_hours} hour(s) ago" else - echo "✗ Settings file: missing" + echo "[ERROR] Settings file: missing" fi echo "" @@ -289,35 +290,35 @@ claude-token-status() { local stored_hash stored_hash=$(cat "$token_hash_file" 2>/dev/null || echo "") if [ "$current_hash" = "$stored_hash" ] && [ -n "$current_hash" ]; then - echo "✓ Token: matches stored hash (up to date)" + echo "[OK] Token: matches stored hash (up to date)" else - echo "⚠ Token: differs from stored hash (needs refresh)" + echo "[WARN] Token: differs from stored hash (needs refresh)" echo " Run: claude-refresh-token" fi else - echo "ℹ Token hash: not stored (will be created on next refresh)" + echo "[INFO] Token hash: not stored (will be created on next refresh)" fi echo "" # Check cron job if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then - echo "✓ Auto-refresh: enabled (hourly cron job)" + echo "[OK] Auto-refresh: enabled (hourly cron job)" else - echo "ℹ Auto-refresh: disabled" + echo "[INFO] Auto-refresh: disabled" echo " Enable with: claude-setup-token-refresh" fi } claude-tracing-enable() { if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then - echo "⚠ DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + echo "[WARN] DATABRICKS_TOKEN and DATABRICKS_HOST must be set" echo " On Databricks clusters, these should be automatically available" return 1 fi if ! command -v mlflow >/dev/null 2>&1; then - echo "⚠ MLflow is not installed" + echo "[WARN] MLflow is not installed" return 1 fi @@ -329,16 +330,16 @@ try: exp = mlflow.get_experiment_by_name("$MLFLOW_EXPERIMENT_NAME") if not exp: mlflow.create_experiment("$MLFLOW_EXPERIMENT_NAME") - print("✓ Created MLflow experiment: $MLFLOW_EXPERIMENT_NAME") + print("[OK] Created MLflow experiment: $MLFLOW_EXPERIMENT_NAME") else: - print("✓ Using existing MLflow experiment: $MLFLOW_EXPERIMENT_NAME") + print("[OK] Using existing MLflow experiment: $MLFLOW_EXPERIMENT_NAME") except Exception as e: - print(f"⚠ Could not setup experiment: {e}") + print(f"[WARN] Could not setup experiment: {e}") MLFLOW_SETUP # Enable autologging mlflow autolog claude "${1:-.}" -u databricks -n "$MLFLOW_EXPERIMENT_NAME" - echo "✓ Claude Code MLflow tracing enabled" + echo "[OK] Claude Code MLflow tracing enabled" } claude-tracing-status() { @@ -356,15 +357,15 @@ check-claude() { # Check PATH echo "PATH includes:" - echo "$PATH" | tr ':' '\n' | grep -E "(claude|local/bin)" || echo " ⚠ No Claude paths found in PATH" + echo "$PATH" | tr ':' '\n' | grep -E "(claude|local/bin)" || echo " [WARN] No Claude paths found in PATH" echo "" # Check Claude if command -v claude >/dev/null 2>&1; then - echo "✓ Claude Code CLI: $(which claude)" + echo "[OK] Claude Code CLI: $(which claude)" claude --version 2>&1 | head -1 || echo " (version check failed)" else - echo "✗ Claude Code CLI: not found" + echo "[ERROR] Claude Code CLI: not found" [ -f "$HOME/.claude/bin/claude" ] && echo " Binary exists at: $HOME/.claude/bin/claude" [ -f "$HOME/.local/bin/claude" ] && echo " Binary exists at: $HOME/.local/bin/claude" fi @@ -373,29 +374,29 @@ check-claude() { # Check configs echo "Configuration files:" if [ -f "$HOME/.claude/settings.json" ]; then - echo " ✓ Claude settings: $HOME/.claude/settings.json" + echo " [OK] Claude settings: $HOME/.claude/settings.json" echo " Preview: $(head -3 $HOME/.claude/settings.json | tail -1)" else - echo " ✗ Claude settings: missing" + echo " [ERROR] Claude settings: missing" fi echo "" # Check environment echo "Environment variables:" - [ -n "$DATABRICKS_HOST" ] && echo " ✓ DATABRICKS_HOST: ${DATABRICKS_HOST}" || echo " ✗ DATABRICKS_HOST: not set" - [ -n "$DATABRICKS_TOKEN" ] && echo " ✓ DATABRICKS_TOKEN: ${DATABRICKS_TOKEN:0:20}..." || echo " ✗ DATABRICKS_TOKEN: not set" - [ -n "$ANTHROPIC_API_KEY" ] && echo " ✓ ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:0:20}..." || echo " ✗ ANTHROPIC_API_KEY: not set" - [ -n "$ANTHROPIC_AUTH_TOKEN" ] && echo " ✓ ANTHROPIC_AUTH_TOKEN: ${ANTHROPIC_AUTH_TOKEN:0:20}..." || echo " ✗ ANTHROPIC_AUTH_TOKEN: not set" - [ -n "$ANTHROPIC_BASE_URL" ] && echo " ✓ ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL}" || echo " ✗ ANTHROPIC_BASE_URL: not set" - [ -n "$ANTHROPIC_MODEL" ] && echo " ✓ ANTHROPIC_MODEL: ${ANTHROPIC_MODEL}" || echo " ✗ ANTHROPIC_MODEL: not set" - [ -n "$ANTHROPIC_CUSTOM_HEADERS" ] && echo " ✓ ANTHROPIC_CUSTOM_HEADERS: ${ANTHROPIC_CUSTOM_HEADERS}" || echo " ✗ ANTHROPIC_CUSTOM_HEADERS: not set" + [ -n "$DATABRICKS_HOST" ] && echo " [OK] DATABRICKS_HOST: ${DATABRICKS_HOST}" || echo " [ERROR] DATABRICKS_HOST: not set" + [ -n "$DATABRICKS_TOKEN" ] && echo " [OK] DATABRICKS_TOKEN: ${DATABRICKS_TOKEN:0:20}..." || echo " [ERROR] DATABRICKS_TOKEN: not set" + [ -n "$ANTHROPIC_API_KEY" ] && echo " [OK] ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:0:20}..." || echo " [ERROR] ANTHROPIC_API_KEY: not set" + [ -n "$ANTHROPIC_AUTH_TOKEN" ] && echo " [OK] ANTHROPIC_AUTH_TOKEN: ${ANTHROPIC_AUTH_TOKEN:0:20}..." || echo " [ERROR] ANTHROPIC_AUTH_TOKEN: not set" + [ -n "$ANTHROPIC_BASE_URL" ] && echo " [OK] ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL}" || echo " [ERROR] ANTHROPIC_BASE_URL: not set" + [ -n "$ANTHROPIC_MODEL" ] && echo " [OK] ANTHROPIC_MODEL: ${ANTHROPIC_MODEL}" || echo " [ERROR] ANTHROPIC_MODEL: not set" + [ -n "$ANTHROPIC_CUSTOM_HEADERS" ] && echo " [OK] ANTHROPIC_CUSTOM_HEADERS: ${ANTHROPIC_CUSTOM_HEADERS}" || echo " [ERROR] ANTHROPIC_CUSTOM_HEADERS: not set" echo "" # Check MLflow if command -v mlflow >/dev/null 2>&1; then - echo "✓ MLflow: $(mlflow --version 2>&1)" + echo "[OK] MLflow: $(mlflow --version 2>&1)" else - echo "✗ MLflow: not found" + echo "[ERROR] MLflow: not found" fi echo "" @@ -403,10 +404,10 @@ check-claude() { echo "Testing Claude CLI authentication:" if command -v claude >/dev/null 2>&1; then if [ -n "$ANTHROPIC_API_KEY" ] || [ -n "$ANTHROPIC_AUTH_TOKEN" ]; then - echo " ✓ Authentication configured via environment variables" + echo " [OK] Authentication configured via environment variables" echo " Test with: echo 'what is 1+1?' | claude --print" else - echo " ⚠ ANTHROPIC_API_KEY/ANTHROPIC_AUTH_TOKEN not set" + echo " [WARN] ANTHROPIC_API_KEY/ANTHROPIC_AUTH_TOKEN not set" echo " Run: source ~/.bashrc" fi fi @@ -417,10 +418,10 @@ check-claude() { local venv_path venv_path=$(claude-vscode-env 2>/dev/null) if [ $? -eq 0 ] && [ -n "$venv_path" ]; then - echo " ✓ Python virtual environment: $venv_path" + echo " [OK] Python virtual environment: $venv_path" echo " Run 'claude-vscode-setup' for setup instructions" else - echo " ℹ Run 'claude-vscode-setup' for Remote SSH setup guide" + echo " [INFO] Run 'claude-vscode-setup' for Remote SSH setup guide" fi echo "" @@ -452,7 +453,7 @@ claude-vscode-env() { if [ -n "$python_envs" ]; then echo "$python_envs" else - echo "⚠ DATABRICKS_VIRTUAL_ENV not set and pythonEnv-* not found" + echo "[WARN] DATABRICKS_VIRTUAL_ENV not set and pythonEnv-* not found" echo " Try: echo \$DATABRICKS_VIRTUAL_ENV" return 1 fi @@ -468,7 +469,7 @@ claude-vscode-setup() { echo "" echo "2. Configure Default Extensions" echo " Open Command Palette (Cmd+Shift+P / Ctrl+Shift+P):" - echo " → Remote-SSH: Settings" + echo " -> Remote-SSH: Settings" echo "" echo " Or edit settings.json and add:" echo "" @@ -480,7 +481,7 @@ claude-vscode-setup() { VSCODE_SETTINGS echo "" echo "3. Connect to Cluster" - echo " - Command Palette → Remote-SSH: Connect to Host" + echo " - Command Palette -> Remote-SSH: Connect to Host" echo " - Enter your cluster SSH connection details" echo "" echo "4. Select Python Interpreter" @@ -495,17 +496,17 @@ VSCODE_SETTINGS echo " $venv_path" echo "" echo " Then in VS Code/Cursor:" - echo " - Command Palette → Python: Select Interpreter" + echo " - Command Palette -> Python: Select Interpreter" echo " - Paste the path above or browse to it" else echo " Run 'echo \$DATABRICKS_VIRTUAL_ENV' to find the path" fi echo "" echo "5. Important Notes" - echo " • IPYNB notebooks and *.py Databricks notebooks have access to" + echo " * IPYNB notebooks and *.py Databricks notebooks have access to" echo " Databricks globals (dbutils, spark, etc.)" - echo " • Regular Python *.py files do NOT have access to Databricks globals" - echo " • Always select the pythonEnv-xxx interpreter for full Databricks" + echo " * Regular Python *.py files do NOT have access to Databricks globals" + echo " * Always select the pythonEnv-xxx interpreter for full Databricks" echo " Runtime library access" echo "" echo "6. Verify Setup" @@ -520,31 +521,31 @@ claude-vscode-check() { local venv_path venv_path=$(claude-vscode-env 2>/dev/null) if [ $? -eq 0 ] && [ -n "$venv_path" ]; then - echo "✓ Python Virtual Environment:" + echo "[OK] Python Virtual Environment:" echo " $venv_path" if [ -d "$venv_path/bin" ]; then - echo " ✓ Virtual environment directory exists" + echo " [OK] Virtual environment directory exists" if [ -f "$venv_path/bin/python" ]; then - echo " ✓ Python executable found" + echo " [OK] Python executable found" echo " Python version: $($venv_path/bin/python --version 2>&1 || echo 'unknown')" else - echo " ⚠ Python executable not found" + echo " [WARN] Python executable not found" fi else - echo " ⚠ Virtual environment directory not found" + echo " [WARN] Virtual environment directory not found" fi else - echo "✗ Python Virtual Environment: Not found" + echo "[ERROR] Python Virtual Environment: Not found" echo " Run: echo \$DATABRICKS_VIRTUAL_ENV" fi echo "" # Check for Python if command -v python3 >/dev/null 2>&1; then - echo "✓ Python3 available: $(which python3)" + echo "[OK] Python3 available: $(which python3)" echo " Version: $(python3 --version 2>&1)" else - echo "✗ Python3 not found in PATH" + echo "[ERROR] Python3 not found in PATH" fi echo "" @@ -564,16 +565,16 @@ for lib in libraries: missing.append(lib) if found: - print(f" ✓ Available: {', '.join(found)}") + print(f" [OK] Available: {', '.join(found)}") if missing: - print(f" ⚠ Missing: {', '.join(missing)}") + print(f" [WARN] Missing: {', '.join(missing)}") # Check for Databricks globals (only available in notebooks) try: import dbutils - print(" ✓ dbutils available (notebook context)") + print(" [OK] dbutils available (notebook context)") except: - print(" ℹ dbutils not available (normal for .py files)") + print(" [INFO] dbutils not available (normal for .py files)") PYTHON_CHECK echo "" @@ -607,7 +608,7 @@ claude-vscode-config() { echo " $venv_path/bin/python" echo "" echo "To set this in VS Code/Cursor:" - echo " 1. Command Palette → Python: Select Interpreter" + echo " 1. Command Palette -> Python: Select Interpreter" echo " 2. Enter interpreter path: $venv_path/bin/python" else echo "To find Python interpreter path, run:" @@ -618,10 +619,71 @@ claude-vscode-config() { EOF sed -i "s|WS_PH|$W|g; s|EXP_PH|$E|g" "$HOME/.bashrc" - log "✓ Bashrc helpers added" + log "[OK] Bashrc helpers added" log " Experiment: $E" } +# Install Databricks skills for Claude Code +install_databricks_skills() { + local skills_dir="$HOME/.claude/skills" + local repo_url="https://raw.githubusercontent.com/databricks-solutions/ai-dev-kit/main/databricks-skills" + + # Core skills to install (curated list for most common use cases) + local core_skills=( + "databricks-config" + "databricks-python-sdk" + "databricks-unity-catalog" + "databricks-jobs" + "asset-bundles" + "databricks-app-python" + "model-serving" + "mlflow-evaluation" + "aibi-dashboards" + "spark-declarative-pipelines" + ) + + log "Installing Databricks skills for Claude Code..." + + # Create skills directory + mkdir -p "$skills_dir" + + local installed=0 + local failed=0 + + for skill in "${core_skills[@]}"; do + local skill_dir="$skills_dir/$skill" + + # Skip if already exists + if [ -d "$skill_dir" ] && [ -f "$skill_dir/SKILL.md" ]; then + log " [INFO] Skill '$skill' already installed" + installed=$((installed + 1)) + continue + fi + + # Create skill directory + mkdir -p "$skill_dir" + + # Download SKILL.md (required) + if curl -sSL -f "${repo_url}/${skill}/SKILL.md" -o "$skill_dir/SKILL.md" 2>>$L; then + log " [OK] Installed skill: $skill" + installed=$((installed + 1)) + else + log " [WARN] Failed to download skill: $skill" + rm -rf "$skill_dir" + failed=$((failed + 1)) + fi + done + + if [ $installed -gt 0 ]; then + log "[OK] Databricks skills installed: $installed skills" + [ $failed -gt 0 ] && log "[WARN] Failed to install: $failed skills" + return 0 + else + log "[WARN] No Databricks skills installed" + return 1 + fi +} + # Main installation main() { log "Starting installation..." @@ -630,42 +692,55 @@ main() { log "Installing system dependencies..." if sudo apt-get update -qq -y &>>$L; then if sudo apt-get install -y -qq curl git jq &>>$L; then - log "✓ System dependencies installed (curl, git, jq)" + log "[OK] System dependencies installed (curl, git, jq)" else - log "⚠ Some system dependencies failed to install" + log "[WARN] Some system dependencies failed to install" fi else - log "⚠ apt-get update failed" + log "[WARN] apt-get update failed" fi # Install MLflow with Databricks support log "Installing MLflow with Databricks support..." if pip install --quiet --upgrade "mlflow[databricks]>=3.4" &>>$L; then - log "✓ MLflow installed successfully" + log "[OK] MLflow installed successfully" else - log "⚠ MLflow installation failed (tracing features will not work)" + log "[WARN] MLflow installation failed (tracing features will not work)" fi # Install tools (continue even if some fail) - install_nodejs || log "⚠ Node.js installation skipped or failed" - install_claude || log "⚠ Claude Code installation skipped or failed" + install_nodejs || log "[WARN] Node.js installation skipped or failed" + install_claude || log "[WARN] Claude Code installation skipped or failed" + + # Install Databricks skills for Claude Code + install_databricks_skills || log "[WARN] Databricks skills installation incomplete" # Configure tools if setup_bashrc; then - log "✓ Bashrc configuration completed" + log "[OK] Bashrc configuration completed" else - log "⚠ Bashrc configuration failed" + log "[WARN] Bashrc configuration failed" fi log "" log "=== Installation Summary ===" log "Installation complete. Full log: $L" log "" + log "Installed components:" + log " - Claude Code CLI" + log " - Node.js runtime" + log " - MLflow with Databricks support" + log " - Databricks skills (patterns and best practices)" + log "" log "Next steps (on cluster login):" log " 1. Run: source ~/.bashrc" log " 2. Verify: check-claude" log " 3. Use: claude command" log "" + log "Databricks skills installed in: ~/.claude/skills/" + log "Skills available: databricks-config, python-sdk, unity-catalog," + log " jobs, asset-bundles, apps, model-serving, mlflow, dashboards, pipelines" + log "" log "Helper commands:" log " - check-claude: Verify installation status" log " - claude-debug: Show Claude CLI configuration details" From 9adaa65da8d3e8f53b160be51ea07a2d76d4dc04 Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Tue, 3 Feb 2026 17:14:44 +1100 Subject: [PATCH 12/14] feat: Add network dependency checker script Add check-network-deps.sh to verify connectivity to all required domains before running the Claude Code installer. This helps diagnose network and firewall issues in restricted environments. Checked domains (10 total): - claude.ai - CLI installer script - api.anthropic.com - Claude CLI binary download - deb.nodesource.com - Node.js repository - archive.ubuntu.com - APT packages (x86_64) - ports.ubuntu.com - APT packages (ARM64) - registry.npmjs.org - NPM packages - pypi.org - Python package index - files.pythonhosted.org - Python package downloads - raw.githubusercontent.com - Databricks skills - storage.googleapis.com - Binary downloads Features: - Color-coded output (green OK / red FAIL) - DNS resolution check before HTTP check - 5-second connect timeout - --verbose flag for detailed HTTP status codes - Exit code 0 for success, 1 for failures - Troubleshooting tips when failures occur Co-Authored-By: Claude Opus 4.5 --- .../scripts/README.md | 72 +++++- .../scripts/check-network-deps.sh | 239 ++++++++++++++++++ 2 files changed, 304 insertions(+), 7 deletions(-) create mode 100755 modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh diff --git a/modules/adb-coding-assistants-cluster/scripts/README.md b/modules/adb-coding-assistants-cluster/scripts/README.md index 7547724d..55707dbd 100644 --- a/modules/adb-coding-assistants-cluster/scripts/README.md +++ b/modules/adb-coding-assistants-cluster/scripts/README.md @@ -190,22 +190,29 @@ claude < prompt.txt claude --stream < task.md ``` -## Internet Dependencies (Online Mode) +## Internet dependencies (online mode) The online installer requires access to: | Domain | Purpose | |--------|---------| -| `claude.ai` | Claude CLI installer | +| `claude.ai` | Claude CLI installer script | +| `api.anthropic.com` | Claude CLI binary download | | `deb.nodesource.com` | Node.js repository | -| `*.ubuntu.com` | System packages | -| `pypi.org` / `files.pythonhosted.org` | Python packages | +| `archive.ubuntu.com` | APT packages (x86_64) | +| `ports.ubuntu.com` | APT packages (ARM64) | | `registry.npmjs.org` | NPM packages | +| `pypi.org` | Python package index | +| `files.pythonhosted.org` | Python package downloads | +| `raw.githubusercontent.com` | Databricks skills | +| `storage.googleapis.com` | Binary downloads | | `${DATABRICKS_HOST}` | Databricks API endpoints | -## Firewall Configuration +> **Tip**: Run `./scripts/check-network-deps.sh` to verify all dependencies are accessible before installation. -If using a firewall, allow HTTPS (443) to these domains, or use the offline installation method. +## Firewall configuration + +If using a firewall, allow HTTPS (443) and HTTP (80) to these domains, or use the offline installation method. ## Environment Variables @@ -225,6 +232,56 @@ The installer supports: - ✅ **amd64** (x86_64) - Default - ✅ **arm64** (aarch64) - Auto-detected +## Network dependency checker + +Before installation, you can verify that all required domains are accessible using the network dependency checker: + +```bash +# Standard check +./scripts/check-network-deps.sh + +# Detailed output with HTTP status codes +./scripts/check-network-deps.sh --verbose +``` + +Example output: +``` +=== Claude Code Network Dependency Check === + +Checking required domains... + +[OK] claude.ai +[OK] api.anthropic.com +[OK] deb.nodesource.com +[OK] archive.ubuntu.com +[OK] ports.ubuntu.com +[OK] registry.npmjs.org +[OK] pypi.org +[OK] files.pythonhosted.org +[OK] raw.githubusercontent.com +[OK] storage.googleapis.com + +---------------------------------------- +Result: 10/10 dependencies reachable + +SUCCESS: All dependencies are accessible +``` + +If any dependencies fail, the script provides troubleshooting guidance: +``` +[OK] claude.ai +[FAIL] deb.nodesource.com - Connection timed out +... +Result: 9/10 dependencies reachable + +FAILED: Some dependencies are not accessible + +Troubleshooting tips: + - Check firewall rules allow HTTPS (443) to the failed domains + - Verify proxy settings if behind a corporate proxy + - For air-gapped environments, use the offline installation module +``` + ## Troubleshooting ### Installation fails during cluster startup @@ -256,11 +313,12 @@ check-claude claude-debug ``` -## File Structure +## File structure ``` scripts/ ├── install-claude.sh # Online installer +├── check-network-deps.sh # Network dependency checker └── README.md # This file ``` diff --git a/modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh b/modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh new file mode 100755 index 00000000..13aede18 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh @@ -0,0 +1,239 @@ +#!/bin/bash +# +# Network Dependency Checker for Claude Code Installation +# +# Verifies connectivity to all required domains before running install-claude.sh. +# Run this script to diagnose network/firewall issues in restricted environments. +# +# Usage: +# ./check-network-deps.sh # Standard check +# ./check-network-deps.sh --verbose # Detailed output +# + +set -euo pipefail + +# ============================================================================ +# Configuration +# ============================================================================ + +CONNECT_TIMEOUT=5 +VERBOSE=false + +# Color codes (disabled if not a terminal) +if [[ -t 1 ]]; then + GREEN='\033[0;32m' + RED='\033[0;31m' + YELLOW='\033[0;33m' + BOLD='\033[1m' + NC='\033[0m' # No Color +else + GREEN='' + RED='' + YELLOW='' + BOLD='' + NC='' +fi + +# Dependencies to check: "domain|purpose|test_url" +DEPENDENCIES=( + "claude.ai|CLI installer|https://claude.ai/install.sh" + "api.anthropic.com|Claude CLI binary|https://api.anthropic.com/" + "deb.nodesource.com|Node.js repo|https://deb.nodesource.com/setup_20.x" + "archive.ubuntu.com|APT packages (x86)|http://archive.ubuntu.com/ubuntu/" + "ports.ubuntu.com|APT packages (ARM)|http://ports.ubuntu.com/ubuntu-ports/" + "registry.npmjs.org|NPM packages|https://registry.npmjs.org/" + "pypi.org|Python packages|https://pypi.org/simple/mlflow/" + "files.pythonhosted.org|Package downloads|https://files.pythonhosted.org/" + "raw.githubusercontent.com|Databricks skills|https://raw.githubusercontent.com/databricks-solutions/ai-dev-kit/main/README.md" + "storage.googleapis.com|Binary downloads|https://storage.googleapis.com/" +) + +# ============================================================================ +# Functions +# ============================================================================ + +usage() { + cat </dev/null; then + host "$domain" &>/dev/null + elif command -v nslookup &>/dev/null; then + nslookup "$domain" &>/dev/null + elif command -v getent &>/dev/null; then + getent hosts "$domain" &>/dev/null + else + # Fall back to ping for DNS resolution + ping -c 1 -W 2 "$domain" &>/dev/null + fi +} + +check_url() { + local url=$1 + local http_code + + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + --connect-timeout "$CONNECT_TIMEOUT" \ + --max-time $((CONNECT_TIMEOUT * 2)) \ + -L "$url" 2>/dev/null || echo "000") + + echo "$http_code" +} + +check_dependency() { + local entry=$1 + local domain purpose test_url + + IFS='|' read -r domain purpose test_url <<< "$entry" + + # Check DNS first + if ! check_dns "$domain"; then + log_fail "$domain - DNS resolution failed" + log_verbose "Purpose: $purpose" + log_verbose "Test URL: $test_url" + return 1 + fi + + # Check HTTP connectivity + local http_code + http_code=$(check_url "$test_url") + + if [[ "$http_code" =~ ^(2[0-9]{2}|3[0-9]{2})$ ]]; then + log_ok "$domain" + log_verbose "Purpose: $purpose" + log_verbose "HTTP status: $http_code" + log_verbose "Test URL: $test_url" + return 0 + else + case "$http_code" in + 000) + log_fail "$domain - Connection timed out" + ;; + 400) + # 400 is common for API endpoints at root - domain is reachable + log_ok "$domain" + log_verbose "Purpose: $purpose" + log_verbose "HTTP status: $http_code (API endpoint - root returns 400)" + log_verbose "Test URL: $test_url" + return 0 + ;; + 403) + log_fail "$domain - Access forbidden (HTTP 403)" + ;; + 404) + # 404 means domain is reachable, just URL changed + log_ok "$domain" + log_verbose "Purpose: $purpose" + log_verbose "HTTP status: $http_code (domain reachable)" + log_verbose "Test URL: $test_url" + return 0 + ;; + *) + log_fail "$domain - HTTP $http_code" + ;; + esac + log_verbose "Purpose: $purpose" + log_verbose "Test URL: $test_url" + return 1 + fi +} + +# ============================================================================ +# Main +# ============================================================================ + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --verbose|-v) + VERBOSE=true + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +# Check for curl +if ! command -v curl &>/dev/null; then + echo "Error: curl is required but not installed" + exit 1 +fi + +echo "" +echo -e "${BOLD}=== Claude Code Network Dependency Check ===${NC}" +echo "" +echo "Checking required domains..." +echo "" + +pass_count=0 +fail_count=0 +total=${#DEPENDENCIES[@]} + +for dep in "${DEPENDENCIES[@]}"; do + if check_dependency "$dep"; then + ((pass_count++)) + else + ((fail_count++)) + fi +done + +echo "" +echo "----------------------------------------" +echo -e "Result: ${BOLD}${pass_count}/${total}${NC} dependencies reachable" + +if [[ $fail_count -gt 0 ]]; then + echo "" + echo -e "${RED}FAILED: Some dependencies are not accessible${NC}" + echo "" + echo "Troubleshooting tips:" + echo " - Check firewall rules allow HTTPS (443) to the failed domains" + echo " - Verify proxy settings if behind a corporate proxy" + echo " - For air-gapped environments, use the offline installation module" + echo "" + exit 1 +else + echo "" + echo -e "${GREEN}SUCCESS: All dependencies are accessible${NC}" + echo "" + exit 0 +fi From e051289a7575975af4b058fa41fcecbd3b73d7c9 Mon Sep 17 00:00:00 2001 From: dgokeeffe Date: Tue, 3 Feb 2026 17:17:12 +1100 Subject: [PATCH 13/14] fix: Correct network dependency descriptions - Remove api.anthropic.com (not used during installation) - Clarify storage.googleapis.com hosts Claude CLI binaries - The Claude installer downloads binaries from GCS bucket: storage.googleapis.com/claude-code-dist-*/claude-code-releases Co-Authored-By: Claude Opus 4.5 --- .../adb-coding-assistants-cluster/scripts/README.md | 10 ++++------ .../scripts/check-network-deps.sh | 5 ++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/modules/adb-coding-assistants-cluster/scripts/README.md b/modules/adb-coding-assistants-cluster/scripts/README.md index 55707dbd..ae644a3a 100644 --- a/modules/adb-coding-assistants-cluster/scripts/README.md +++ b/modules/adb-coding-assistants-cluster/scripts/README.md @@ -197,7 +197,7 @@ The online installer requires access to: | Domain | Purpose | |--------|---------| | `claude.ai` | Claude CLI installer script | -| `api.anthropic.com` | Claude CLI binary download | +| `storage.googleapis.com` | Claude CLI binaries (GCS bucket) | | `deb.nodesource.com` | Node.js repository | | `archive.ubuntu.com` | APT packages (x86_64) | | `ports.ubuntu.com` | APT packages (ARM64) | @@ -205,7 +205,6 @@ The online installer requires access to: | `pypi.org` | Python package index | | `files.pythonhosted.org` | Python package downloads | | `raw.githubusercontent.com` | Databricks skills | -| `storage.googleapis.com` | Binary downloads | | `${DATABRICKS_HOST}` | Databricks API endpoints | > **Tip**: Run `./scripts/check-network-deps.sh` to verify all dependencies are accessible before installation. @@ -251,7 +250,7 @@ Example output: Checking required domains... [OK] claude.ai -[OK] api.anthropic.com +[OK] storage.googleapis.com [OK] deb.nodesource.com [OK] archive.ubuntu.com [OK] ports.ubuntu.com @@ -259,10 +258,9 @@ Checking required domains... [OK] pypi.org [OK] files.pythonhosted.org [OK] raw.githubusercontent.com -[OK] storage.googleapis.com ---------------------------------------- -Result: 10/10 dependencies reachable +Result: 9/9 dependencies reachable SUCCESS: All dependencies are accessible ``` @@ -272,7 +270,7 @@ If any dependencies fail, the script provides troubleshooting guidance: [OK] claude.ai [FAIL] deb.nodesource.com - Connection timed out ... -Result: 9/10 dependencies reachable +Result: 8/9 dependencies reachable FAILED: Some dependencies are not accessible diff --git a/modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh b/modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh index 13aede18..3ef4bec2 100755 --- a/modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh +++ b/modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh @@ -36,8 +36,8 @@ fi # Dependencies to check: "domain|purpose|test_url" DEPENDENCIES=( - "claude.ai|CLI installer|https://claude.ai/install.sh" - "api.anthropic.com|Claude CLI binary|https://api.anthropic.com/" + "claude.ai|CLI installer script|https://claude.ai/install.sh" + "storage.googleapis.com|Claude CLI binaries|https://storage.googleapis.com/" "deb.nodesource.com|Node.js repo|https://deb.nodesource.com/setup_20.x" "archive.ubuntu.com|APT packages (x86)|http://archive.ubuntu.com/ubuntu/" "ports.ubuntu.com|APT packages (ARM)|http://ports.ubuntu.com/ubuntu-ports/" @@ -45,7 +45,6 @@ DEPENDENCIES=( "pypi.org|Python packages|https://pypi.org/simple/mlflow/" "files.pythonhosted.org|Package downloads|https://files.pythonhosted.org/" "raw.githubusercontent.com|Databricks skills|https://raw.githubusercontent.com/databricks-solutions/ai-dev-kit/main/README.md" - "storage.googleapis.com|Binary downloads|https://storage.googleapis.com/" ) # ============================================================================ From d990f734e986b0ec32ffb8439516bdd390b96316 Mon Sep 17 00:00:00 2001 From: David O'Keeffe Date: Mon, 20 Jul 2026 20:08:52 +1000 Subject: [PATCH 14/14] feat: add aws-uc-external-location-file-events module Add an AWS module (and example) that creates a Unity Catalog storage credential and external location(s) with automatic managed SQS file events. This is the AWS counterpart to the existing Azure adb-uc-external-location-file-events module. The module encodes the two non-obvious requirements for this pattern: - a self-assuming IAM role (UCMasterRole trust gated by the Databricks account external ID, plus an AllowSelfAssume sts:AssumeRole permission on the role itself); and - the csms-*-scoped sns/sqs/s3 bucket-notification permissions required for automatic managed file events, without which file events silently fail to provision. Includes a time_sleep to let IAM propagate before UC validation, optional UC grants, and a paired example under examples/. --- README.md | 1 + .../README.md | 88 +++++++ .../main.tf | 43 ++++ .../outputs.tf | 19 ++ .../providers.tf | 11 + .../terraform.tfvars.example | 12 + .../variables.tf | 51 ++++ .../versions.tf | 18 ++ .../Makefile | 7 + .../README.md | 155 ++++++++++++ .../main.tf | 220 ++++++++++++++++++ .../outputs.tf | 44 ++++ .../providers.tf | 18 ++ .../variables.tf | 127 ++++++++++ .../versions.tf | 18 ++ 15 files changed, 832 insertions(+) create mode 100644 examples/aws-uc-external-location-file-events/README.md create mode 100644 examples/aws-uc-external-location-file-events/main.tf create mode 100644 examples/aws-uc-external-location-file-events/outputs.tf create mode 100644 examples/aws-uc-external-location-file-events/providers.tf create mode 100644 examples/aws-uc-external-location-file-events/terraform.tfvars.example create mode 100644 examples/aws-uc-external-location-file-events/variables.tf create mode 100644 examples/aws-uc-external-location-file-events/versions.tf create mode 100644 modules/aws-uc-external-location-file-events/Makefile create mode 100644 modules/aws-uc-external-location-file-events/README.md create mode 100644 modules/aws-uc-external-location-file-events/main.tf create mode 100644 modules/aws-uc-external-location-file-events/outputs.tf create mode 100644 modules/aws-uc-external-location-file-events/providers.tf create mode 100644 modules/aws-uc-external-location-file-events/variables.tf create mode 100644 modules/aws-uc-external-location-file-events/versions.tf diff --git a/README.md b/README.md index 86baaf9b..ac638e34 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ The folder `modules` contains the following Terraform modules : | AWS | [aws-databricks-workspace](modules/aws-databricks-workspace/) | Provisioning AWS Databricks E2 Workspace using pre-created AWS Infra | | AWS | [aws-workspace-with-firewall](modules/aws-workspace-with-firewall/) | Provisioning AWS Databricks E2 with an AWS Firewall | | AWS | [aws-exfiltration-protection](modules/aws-exfiltration-protection/) | An implementation of [Data Exfiltration Protection on AWS](https://www.databricks.com/blog/2021/02/02/data-exfiltration-protection-with-databricks-on-aws.html) | +| AWS | [aws-uc-external-location-file-events](modules/aws-uc-external-location-file-events/) | Unity Catalog storage credential (self-assuming IAM role) and external location(s) with automatic managed SQS file events | | AWS | aws-workspace-with-private-link | Coming soon | | GCP | [gcp-sa-provisionning](modules/gcp-sa-provisionning/) | Provisions the identity (SA) with the correct permissions | | GCP | [gcp-workspace-basic](modules/gcp-workspace-basic/) | Provisions a workspace with managed VPC | diff --git a/examples/aws-uc-external-location-file-events/README.md b/examples/aws-uc-external-location-file-events/README.md new file mode 100644 index 00000000..262555e2 --- /dev/null +++ b/examples/aws-uc-external-location-file-events/README.md @@ -0,0 +1,88 @@ +# Example: aws-uc-external-location-file-events + +Creates an S3 bucket and uses the +[`aws-uc-external-location-file-events`](../../modules/aws-uc-external-location-file-events) +module to create a Unity Catalog storage credential (backed by a self-assuming IAM role) and an +external location with **automatic managed file events** (SNS + SQS, `csms-*` prefixed). + +## Prerequisites + +- A Unity Catalog–enabled Databricks workspace whose assigned metastore is the target metastore. +- AWS credentials for the account where the bucket/IAM role live (profile or env vars). +- Databricks auth for that workspace (CLI profile or `DATABRICKS_*` env vars). +- Permission to create IAM roles/policies and UC storage credentials + external locations. + +## Usage + +```bash +cp terraform.tfvars.example terraform.tfvars +# edit terraform.tfvars + +terraform init +terraform plan +terraform apply +``` + +After apply, verify file events with the Unity Catalog storage-credential validation API: + +```bash +databricks api post /api/2.1/unity-catalog/validate-storage-credentials \ + --json '{"storage_credential_name":"","external_location_name":""}' +``` + +`READ_MESSAGE` transitions from `SKIP` ("being provisioned") to `PASS` once Databricks +provisions the managed SNS topic + SQS queue and the S3 bucket notification. + + +## Requirements + +| Name | Version | +| ---- | ------- | +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | >= 5.0 | +| [databricks](#requirement\_databricks) | >= 1.81.1 | +| [time](#requirement\_time) | >= 0.9.0 | + +## Providers + +| Name | Version | +| ---- | ------- | +| [aws](#provider\_aws) | >= 5.0 | + +## Modules + +| Name | Source | Version | +| ---- | ------ | ------- | +| [uc\_external\_location\_file\_events](#module\_uc\_external\_location\_file\_events) | ../../modules/aws-uc-external-location-file-events | n/a | + +## Resources + +| Name | Type | +| ---- | ---- | +| [aws_s3_bucket.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket_public_access_block.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| [bucket\_name](#input\_bucket\_name) | Globally unique S3 bucket name to create for the external location | `string` | n/a | yes | +| [databricks\_account\_id](#input\_databricks\_account\_id) | Databricks account ID (used as the sts:ExternalId in the IAM role trust policy). | `string` | n/a | yes | +| [aws\_profile](#input\_aws\_profile) | AWS CLI profile for the account that owns the bucket. Leave empty to use the default credential chain / env vars. | `string` | `""` | no | +| [aws\_region](#input\_aws\_region) | AWS region for the S3 bucket | `string` | `"us-east-1"` | no | +| [databricks\_profile](#input\_databricks\_profile) | Databricks CLI profile for a workspace assigned to the target metastore. Leave empty to use DATABRICKS\_* env vars. | `string` | `""` | no | +| [external\_location\_name](#input\_external\_location\_name) | Name of the Unity Catalog external location | `string` | `"file_events_landing"` | no | +| [external\_location\_prefix](#input\_external\_location\_prefix) | Prefix within the bucket managed by the external location | `string` | `"landing"` | no | +| [grant\_principal](#input\_grant\_principal) | UC group or user to grant on the credential and external location. Leave empty to skip grants. | `string` | `""` | no | +| [name\_prefix](#input\_name\_prefix) | Prefix for UC object and IAM names | `string` | `"file-events-demo"` | no | + +## Outputs + +| Name | Description | +| ---- | ----------- | +| [bucket\_name](#output\_bucket\_name) | n/a | +| [external\_location\_ids](#output\_external\_location\_ids) | n/a | +| [external\_location\_urls](#output\_external\_location\_urls) | n/a | +| [iam\_role\_arn](#output\_iam\_role\_arn) | n/a | +| [storage\_credential\_name](#output\_storage\_credential\_name) | n/a | + diff --git a/examples/aws-uc-external-location-file-events/main.tf b/examples/aws-uc-external-location-file-events/main.tf new file mode 100644 index 00000000..3f25b628 --- /dev/null +++ b/examples/aws-uc-external-location-file-events/main.tf @@ -0,0 +1,43 @@ +# The caller owns the bucket; the module wires up IAM + Unity Catalog. +resource "aws_s3_bucket" "this" { + bucket = var.bucket_name + force_destroy = true +} + +resource "aws_s3_bucket_public_access_block" "this" { + bucket = aws_s3_bucket.this.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +module "uc_external_location_file_events" { + source = "../../modules/aws-uc-external-location-file-events" + + name_prefix = var.name_prefix + databricks_account_id = var.databricks_account_id + bucket_names = [aws_s3_bucket.this.id] + + external_locations = [ + { + name = var.external_location_name + url = "s3://${aws_s3_bucket.this.id}/${var.external_location_prefix}" + comment = "Landing zone with managed SQS file events" + } + ] + + credential_grants = var.grant_principal == "" ? [] : [ + { + principal = var.grant_principal + privileges = ["CREATE_EXTERNAL_LOCATION", "READ_FILES", "WRITE_FILES"] + } + ] + + location_grants = var.grant_principal == "" ? [] : [ + { + principal = var.grant_principal + privileges = ["BROWSE", "READ_FILES", "WRITE_FILES", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME"] + } + ] +} diff --git a/examples/aws-uc-external-location-file-events/outputs.tf b/examples/aws-uc-external-location-file-events/outputs.tf new file mode 100644 index 00000000..ea9ee8bb --- /dev/null +++ b/examples/aws-uc-external-location-file-events/outputs.tf @@ -0,0 +1,19 @@ +output "bucket_name" { + value = aws_s3_bucket.this.id +} + +output "iam_role_arn" { + value = module.uc_external_location_file_events.iam_role_arn +} + +output "storage_credential_name" { + value = module.uc_external_location_file_events.storage_credential_name +} + +output "external_location_ids" { + value = module.uc_external_location_file_events.external_location_ids +} + +output "external_location_urls" { + value = module.uc_external_location_file_events.external_location_urls +} diff --git a/examples/aws-uc-external-location-file-events/providers.tf b/examples/aws-uc-external-location-file-events/providers.tf new file mode 100644 index 00000000..4a143cec --- /dev/null +++ b/examples/aws-uc-external-location-file-events/providers.tf @@ -0,0 +1,11 @@ +provider "aws" { + region = var.aws_region + profile = var.aws_profile +} + +# Authenticate to an existing UC-enabled workspace whose assigned metastore is the +# one you want these objects in. Prefer `databricks auth login --host ` +# (a CLI profile) or environment variables over inline credentials. +provider "databricks" { + profile = var.databricks_profile +} diff --git a/examples/aws-uc-external-location-file-events/terraform.tfvars.example b/examples/aws-uc-external-location-file-events/terraform.tfvars.example new file mode 100644 index 00000000..acb4b48d --- /dev/null +++ b/examples/aws-uc-external-location-file-events/terraform.tfvars.example @@ -0,0 +1,12 @@ +aws_region = "ap-southeast-2" +aws_profile = "my-aws-profile" +databricks_profile = "my-workspace-profile" + +databricks_account_id = "00000000-0000-0000-0000-000000000000" + +name_prefix = "file-events-demo" +bucket_name = "my-globally-unique-uc-demo-bucket" +external_location_name = "file_events_landing" +external_location_prefix = "landing" + +grant_principal = "data-engineers" diff --git a/examples/aws-uc-external-location-file-events/variables.tf b/examples/aws-uc-external-location-file-events/variables.tf new file mode 100644 index 00000000..b94cac8d --- /dev/null +++ b/examples/aws-uc-external-location-file-events/variables.tf @@ -0,0 +1,51 @@ +variable "aws_region" { + type = string + description = "AWS region for the S3 bucket" + default = "us-east-1" +} + +variable "aws_profile" { + type = string + description = "AWS CLI profile for the account that owns the bucket. Leave empty to use the default credential chain / env vars." + default = "" +} + +variable "databricks_profile" { + type = string + description = "Databricks CLI profile for a workspace assigned to the target metastore. Leave empty to use DATABRICKS_* env vars." + default = "" +} + +variable "databricks_account_id" { + type = string + description = "Databricks account ID (used as the sts:ExternalId in the IAM role trust policy)." +} + +variable "name_prefix" { + type = string + description = "Prefix for UC object and IAM names" + default = "file-events-demo" +} + +variable "bucket_name" { + type = string + description = "Globally unique S3 bucket name to create for the external location" +} + +variable "external_location_name" { + type = string + description = "Name of the Unity Catalog external location" + default = "file_events_landing" +} + +variable "external_location_prefix" { + type = string + description = "Prefix within the bucket managed by the external location" + default = "landing" +} + +variable "grant_principal" { + type = string + description = "UC group or user to grant on the credential and external location. Leave empty to skip grants." + default = "" +} diff --git a/examples/aws-uc-external-location-file-events/versions.tf b/examples/aws-uc-external-location-file-events/versions.tf new file mode 100644 index 00000000..23eeccad --- /dev/null +++ b/examples/aws-uc-external-location-file-events/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + databricks = { + source = "databricks/databricks" + version = ">= 1.81.1" + } + time = { + source = "hashicorp/time" + version = ">= 0.9.0" + } + } +} diff --git a/modules/aws-uc-external-location-file-events/Makefile b/modules/aws-uc-external-location-file-events/Makefile new file mode 100644 index 00000000..653039d8 --- /dev/null +++ b/modules/aws-uc-external-location-file-events/Makefile @@ -0,0 +1,7 @@ +.PHONY: docs test_docs + +docs: + terraform-docs -c ../../.terraform-docs.yml . + +test_docs: + terraform-docs -c ../../.terraform-docs.yml --output-check . diff --git a/modules/aws-uc-external-location-file-events/README.md b/modules/aws-uc-external-location-file-events/README.md new file mode 100644 index 00000000..58027040 --- /dev/null +++ b/modules/aws-uc-external-location-file-events/README.md @@ -0,0 +1,155 @@ +# aws-uc-external-location-file-events + +Creates AWS Unity Catalog **storage credentials** and **external locations** with +[managed file events](https://docs.databricks.com/aws/en/connect/unity-catalog/cloud-storage/manage-external-locations) +enabled via Amazon SNS + SQS (managed SQS). The module builds the +**self-assuming IAM role** and the IAM policy that Unity Catalog requires, including the +extra `sns:*` / `sqs:*` / S3 bucket-notification permissions needed for automatic file-event +setup. + +This is useful for [file arrival triggers](https://docs.databricks.com/aws/en/jobs/file-arrival-triggers) +and other ingestion patterns (for example [Auto Loader](https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/file-events-explained)) +that benefit from cloud storage change notifications. + +This is the AWS counterpart to +[`adb-uc-external-location-file-events`](../adb-uc-external-location-file-events) (Azure). + +## Prerequisites + +- Unity Catalog–enabled Databricks workspace whose assigned metastore is the target metastore + (the Databricks provider should authenticate to that workspace), **or** an account-level + provider with `metastore_id` set. +- One or more existing S3 buckets that back the external locations. This module does **not** + create the buckets — the caller owns them (see the example). +- Permission to create IAM roles/policies in the AWS account that owns the bucket(s). +- Permission to create UC storage credentials and external locations (metastore admin or the + `CREATE STORAGE CREDENTIAL` / `CREATE EXTERNAL LOCATION` privileges). + +## What it creates + +| Resource | Purpose | +| -------- | ------- | +| `aws_iam_policy` | S3 data access on the target bucket(s), `sts:AssumeRole` self-permission, and (when `enable_file_events`) the `csms-*`-scoped SNS/SQS/bucket-notification permissions | +| `aws_iam_role` | Self-assuming role trusted by the UC master role (gated by the Databricks account ID as external ID) | +| `time_sleep` | Waits for IAM propagation before UC validates the credential (avoids transient "non self-assuming" / 403 errors) | +| `databricks_storage_credential` | UC storage credential backed by the IAM role | +| `databricks_external_location` | One per entry in `external_locations`, with automatic managed SQS file events | +| `databricks_grants` | Optional UC grants on the credential and locations | + +## Managed file events (automatic mode) + +When `enable_file_events = true` (default), the IAM policy includes the documented +`ManagedFileEventsSetupStatement` / `ManagedFileEventsListStatement` / +`ManagedFileEventsTeardownStatement` statements, scoped to the target bucket(s) and the +`csms-*` SNS/SQS namespace. Unity Catalog then provisions one SNS topic and one SQS queue +(prefixed `csms-*`) per external location and configures the S3 bucket notification. Without +these permissions, file events silently fail to provision even though the location reports it +as enabled. + +## Example + +```hcl +# The caller owns the bucket. +resource "aws_s3_bucket" "demo" { + bucket = "my-uc-demo-bucket" + force_destroy = true +} + +module "uc_locations" { + source = "../../modules/aws-uc-external-location-file-events" + + name_prefix = "demo" + databricks_account_id = "00000000-0000-0000-0000-000000000000" + bucket_names = [aws_s3_bucket.demo.id] + + external_locations = [ + { + name = "demo_landing" + url = "s3://${aws_s3_bucket.demo.id}/landing" + comment = "Landing zone with managed SQS file events" + } + ] + + location_grants = [ + { + principal = "data-engineers" + privileges = ["BROWSE", "READ_FILES", "WRITE_FILES", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME"] + } + ] +} +``` + +See also [examples/aws-uc-external-location-file-events](../../examples/aws-uc-external-location-file-events). + + +## Requirements + +| Name | Version | +| ---- | ------- | +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | >= 5.0 | +| [databricks](#requirement\_databricks) | >= 1.81.1 | +| [time](#requirement\_time) | >= 0.9.0 | + +## Providers + +| Name | Version | +| ---- | ------- | +| [aws](#provider\_aws) | >= 5.0 | +| [databricks](#provider\_databricks) | >= 1.81.1 | +| [time](#provider\_time) | >= 0.9.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| ---- | ---- | +| [aws_iam_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [databricks_external_location.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/external_location) | resource | +| [databricks_grants.credential](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource | +| [databricks_grants.location](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource | +| [databricks_storage_credential.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/storage_credential) | resource | +| [time_sleep.wait_iam](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| [bucket\_names](#input\_bucket\_names) | Names of the S3 buckets that the storage credential IAM role is granted access to (and, when file events are enabled, allowed to configure notifications on). The buckets must already exist; this module does not create them. | `list(string)` | n/a | yes | +| [databricks\_account\_id](#input\_databricks\_account\_id) | Databricks account ID. Used as the sts:ExternalId in the IAM role trust policy (the Unity Catalog storage credential external ID). | `string` | n/a | yes | +| [external\_locations](#input\_external\_locations) | External locations to create. Each location has managed SQS file events enabled when enable\_file\_events is true. | 
list(object({
    name      = string
    url       = string
    comment   = optional(string, "Managed by Terraform")
    read_only = optional(bool, false)
  }))
| n/a | yes | +| [name\_prefix](#input\_name\_prefix) | Prefix used to derive default names for the IAM role, IAM policy, and storage credential. | `string` | n/a | yes | +| [create\_storage\_credential](#input\_create\_storage\_credential) | When true, create a storage credential backed by the IAM role this module manages. When false, reuse existing\_credential\_name (the IAM role/policy are still managed). | `bool` | `true` | no | +| [credential\_grants](#input\_credential\_grants) | UC grants applied to the storage credential. Defaults to empty (owner-only). | 
list(object({
    principal  = string
    privileges = list(string)
  }))
| `[]` | no | +| [enable\_file\_events](#input\_enable\_file\_events) | Enable automatic managed file events (SNS topic + SQS queue + S3 bucket notification, csms-* prefixed) on every external location. Adds the required sns/sqs/bucket-notification permissions to the IAM policy. | `bool` | `true` | no | +| [existing\_credential\_name](#input\_existing\_credential\_name) | Name of an existing storage credential to reference on the external locations when create\_storage\_credential is false. | `string` | `""` | no | +| [force\_destroy](#input\_force\_destroy) | Force destroy the storage credential and external locations even if dependents exist. | `bool` | `false` | no | +| [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy attached to the role. Defaults to "-policy". | `string` | `""` | no | +| [iam\_propagation\_delay](#input\_iam\_propagation\_delay) | Delay to wait after creating the IAM role/policy before creating the storage credential and external locations, so IAM changes propagate (avoids "non self-assuming" / 403 validation errors). Set to "" to disable the wait (e.g. when the role already exists). | `string` | `"60s"` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name of the IAM role used by the storage credential. Defaults to "-uc". | `string` | `""` | no | +| [location\_grants](#input\_location\_grants) | UC grants applied to every external location.
Recommended privileges for data engineers: BROWSE, READ\_FILES, WRITE\_FILES,
CREATE\_EXTERNAL\_TABLE, CREATE\_EXTERNAL\_VOLUME. | 
list(object({
    principal  = string
    privileges = list(string)
  }))
| `[]` | no | +| [storage\_credential\_comment](#input\_storage\_credential\_comment) | Comment applied to the created storage credential. | `string` | `"Storage credential for external locations with file events. Managed by Terraform."` | no | +| [storage\_credential\_name](#input\_storage\_credential\_name) | Name for the created storage credential. Defaults to "-storage-credential". | `string` | `""` | no | +| [tags](#input\_tags) | Tags applied to the IAM role and policy. | `map(string)` | `{}` | no | +| [uc\_master\_role\_arn](#input\_uc\_master\_role\_arn) | ARN of the Unity Catalog AWS master role that assumes the storage credential role. Defaults to the Databricks commercial (non-GovCloud) UC master role. | `string` | `"arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL"` | no | + +## Outputs + +| Name | Description | +| ---- | ----------- | +| [external\_location\_ids](#output\_external\_location\_ids) | Map of external location name to ID | +| [external\_location\_names](#output\_external\_location\_names) | Names of the created external locations | +| [external\_location\_urls](#output\_external\_location\_urls) | Map of external location name to URL | +| [file\_events\_enabled](#output\_file\_events\_enabled) | Whether managed file events are enabled on the external locations | +| [iam\_policy\_arn](#output\_iam\_policy\_arn) | ARN of the IAM policy attached to the role | +| [iam\_role\_arn](#output\_iam\_role\_arn) | ARN of the IAM role trusted by the Unity Catalog storage credential | +| [iam\_role\_name](#output\_iam\_role\_name) | Name of the IAM role backing the storage credential | +| [storage\_credential\_id](#output\_storage\_credential\_id) | ID of the created storage credential (null when reusing an existing credential) | +| [storage\_credential\_name](#output\_storage\_credential\_name) | Name of the storage credential used by the external locations | + diff --git a/modules/aws-uc-external-location-file-events/main.tf b/modules/aws-uc-external-location-file-events/main.tf new file mode 100644 index 00000000..a27febf2 --- /dev/null +++ b/modules/aws-uc-external-location-file-events/main.tf @@ -0,0 +1,220 @@ +data "aws_caller_identity" "current" {} + +locals { + iam_role_name = var.iam_role_name != "" ? var.iam_role_name : "${var.name_prefix}-uc" + iam_policy_name = var.iam_policy_name != "" ? var.iam_policy_name : "${local.iam_role_name}-policy" + storage_credential_name = var.storage_credential_name != "" ? var.storage_credential_name : "${var.name_prefix}-storage-credential" + + role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${local.iam_role_name}" + credential_name = var.create_storage_credential ? databricks_storage_credential.this[0].name : var.existing_credential_name + + bucket_arns = [for b in var.bucket_names : "arn:aws:s3:::${b}"] + bucket_object_arns = [for b in var.bucket_names : "arn:aws:s3:::${b}/*"] + + # Base data-access statements (always present). + base_statements = [ + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + "s3:PutObjectAcl", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetBucketLocation" + ] + Resource = concat(local.bucket_arns, local.bucket_object_arns) + }, + { + Sid = "AllowSelfAssume" + Effect = "Allow" + Action = ["sts:AssumeRole"] + Resource = [local.role_arn] + } + ] + + # Managed file events (Automatic mode): lets Unity Catalog configure S3 bucket + # notifications, create the SNS topic + SQS queue (csms-* prefixed) and subscribe + # the queue to the topic. Scoped to the target buckets and the csms-* namespace. + # Docs: https://docs.databricks.com/aws/en/connect/unity-catalog/cloud-storage/manage-external-locations + file_event_statements = var.enable_file_events ? [ + { + Sid = "ManagedFileEventsSetupStatement" + Effect = "Allow" + Action = [ + "s3:GetBucketNotification", + "s3:PutBucketNotification", + "sns:ListSubscriptionsByTopic", + "sns:GetTopicAttributes", + "sns:SetTopicAttributes", + "sns:CreateTopic", + "sns:TagResource", + "sns:Publish", + "sns:Subscribe", + "sqs:CreateQueue", + "sqs:DeleteMessage", + "sqs:ReceiveMessage", + "sqs:SendMessage", + "sqs:GetQueueUrl", + "sqs:GetQueueAttributes", + "sqs:SetQueueAttributes", + "sqs:TagQueue", + "sqs:ChangeMessageVisibility", + "sqs:PurgeQueue" + ] + Resource = concat(local.bucket_arns, ["arn:aws:sqs:*:*:csms-*", "arn:aws:sns:*:*:csms-*"]) + }, + { + Sid = "ManagedFileEventsListStatement" + Effect = "Allow" + Action = ["sqs:ListQueues", "sqs:ListQueueTags", "sns:ListTopics"] + Resource = ["arn:aws:sqs:*:*:csms-*", "arn:aws:sns:*:*:csms-*"] + }, + { + Sid = "ManagedFileEventsTeardownStatement" + Effect = "Allow" + Action = ["sns:Unsubscribe", "sns:DeleteTopic", "sqs:DeleteQueue"] + Resource = ["arn:aws:sqs:*:*:csms-*", "arn:aws:sns:*:*:csms-*"] + } + ] : [] +} + +# Self-assuming role trust policy required by Unity Catalog storage credentials: +# 1. The UC master role may assume this role, gated by the external ID. +# 2. The role may assume itself (paired with the AllowSelfAssume permission below). +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + type = "AWS" + identifiers = [var.uc_master_role_arn] + } + condition { + test = "StringEquals" + variable = "sts:ExternalId" + values = [var.databricks_account_id] + } + } + + statement { + sid = "ExplicitSelfRoleAssumption" + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + } + condition { + test = "ArnLike" + variable = "aws:PrincipalArn" + values = [local.role_arn] + } + } +} + +resource "aws_iam_policy" "this" { + name = local.iam_policy_name + + policy = jsonencode({ + Version = "2012-10-17" + Id = "${var.bucket_names[0]}-access" + Statement = concat(local.base_statements, local.file_event_statements) + }) + + tags = merge(var.tags, { + Name = local.iam_policy_name + }) +} + +resource "aws_iam_role" "this" { + name = local.iam_role_name + assume_role_policy = data.aws_iam_policy_document.assume_role.json + managed_policy_arns = [aws_iam_policy.this.arn] + + tags = merge(var.tags, { + Name = local.iam_role_name + }) +} + +# Give the IAM role/policy time to propagate before Unity Catalog validates the +# credential and external locations (avoids transient "non self-assuming" / 403s). +resource "time_sleep" "wait_iam" { + count = var.iam_propagation_delay == "" ? 0 : 1 + + create_duration = var.iam_propagation_delay + + triggers = { + role_arn = aws_iam_role.this.arn + policy_arn = aws_iam_policy.this.arn + } +} + +resource "databricks_storage_credential" "this" { + count = var.create_storage_credential ? 1 : 0 + + name = local.storage_credential_name + comment = var.storage_credential_comment + + aws_iam_role { + role_arn = aws_iam_role.this.arn + } + + force_destroy = var.force_destroy ? true : null + + depends_on = [time_sleep.wait_iam] +} + +resource "databricks_external_location" "this" { + for_each = { for loc in var.external_locations : loc.name => loc } + + name = each.value.name + url = each.value.url + credential_name = local.credential_name + comment = each.value.comment + read_only = each.value.read_only + + enable_file_events = var.enable_file_events + dynamic "file_event_queue" { + for_each = var.enable_file_events ? [1] : [] + content { + managed_sqs {} + } + } + + force_destroy = var.force_destroy ? true : null + + depends_on = [ + databricks_storage_credential.this, + time_sleep.wait_iam, + ] +} + +resource "databricks_grants" "credential" { + count = var.create_storage_credential && length(var.credential_grants) > 0 ? 1 : 0 + + storage_credential = databricks_storage_credential.this[0].id + + dynamic "grant" { + for_each = var.credential_grants + content { + principal = grant.value.principal + privileges = grant.value.privileges + } + } +} + +resource "databricks_grants" "location" { + for_each = length(var.location_grants) > 0 ? databricks_external_location.this : {} + + external_location = each.value.id + + dynamic "grant" { + for_each = var.location_grants + content { + principal = grant.value.principal + privileges = grant.value.privileges + } + } +} diff --git a/modules/aws-uc-external-location-file-events/outputs.tf b/modules/aws-uc-external-location-file-events/outputs.tf new file mode 100644 index 00000000..98f7ae4e --- /dev/null +++ b/modules/aws-uc-external-location-file-events/outputs.tf @@ -0,0 +1,44 @@ +output "iam_role_name" { + description = "Name of the IAM role backing the storage credential" + value = aws_iam_role.this.name +} + +output "iam_role_arn" { + description = "ARN of the IAM role trusted by the Unity Catalog storage credential" + value = aws_iam_role.this.arn +} + +output "iam_policy_arn" { + description = "ARN of the IAM policy attached to the role" + value = aws_iam_policy.this.arn +} + +output "storage_credential_name" { + description = "Name of the storage credential used by the external locations" + value = local.credential_name +} + +output "storage_credential_id" { + description = "ID of the created storage credential (null when reusing an existing credential)" + value = try(databricks_storage_credential.this[0].id, null) +} + +output "external_location_names" { + description = "Names of the created external locations" + value = [for loc in databricks_external_location.this : loc.name] +} + +output "external_location_ids" { + description = "Map of external location name to ID" + value = { for k, v in databricks_external_location.this : k => v.id } +} + +output "external_location_urls" { + description = "Map of external location name to URL" + value = { for k, v in databricks_external_location.this : k => v.url } +} + +output "file_events_enabled" { + description = "Whether managed file events are enabled on the external locations" + value = var.enable_file_events +} diff --git a/modules/aws-uc-external-location-file-events/providers.tf b/modules/aws-uc-external-location-file-events/providers.tf new file mode 100644 index 00000000..dced1992 --- /dev/null +++ b/modules/aws-uc-external-location-file-events/providers.tf @@ -0,0 +1,18 @@ +# This module expects: +# - an AWS provider with permission to manage IAM roles/policies for the account +# that owns the S3 bucket(s) backing the external location(s) +# - a Databricks provider authenticated to a UC-enabled workspace whose assigned +# metastore is the one you want the storage credential / external locations in +# (workspace-level auth), or an account-level provider with metastore_id set on +# the resources. +# +# Example: +# +# provider "aws" { +# region = "ap-southeast-2" +# profile = "my-aws-profile" +# } +# +# provider "databricks" { +# profile = "my-workspace-profile" +# } diff --git a/modules/aws-uc-external-location-file-events/variables.tf b/modules/aws-uc-external-location-file-events/variables.tf new file mode 100644 index 00000000..d48668b8 --- /dev/null +++ b/modules/aws-uc-external-location-file-events/variables.tf @@ -0,0 +1,127 @@ +variable "name_prefix" { + type = string + description = "Prefix used to derive default names for the IAM role, IAM policy, and storage credential." +} + +variable "databricks_account_id" { + type = string + description = "Databricks account ID. Used as the sts:ExternalId in the IAM role trust policy (the Unity Catalog storage credential external ID)." +} + +variable "uc_master_role_arn" { + type = string + description = "ARN of the Unity Catalog AWS master role that assumes the storage credential role. Defaults to the Databricks commercial (non-GovCloud) UC master role." + default = "arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL" +} + +variable "bucket_names" { + type = list(string) + description = "Names of the S3 buckets that the storage credential IAM role is granted access to (and, when file events are enabled, allowed to configure notifications on). The buckets must already exist; this module does not create them." + + validation { + condition = length(var.bucket_names) > 0 + error_message = "At least one bucket name is required." + } +} + +variable "external_locations" { + type = list(object({ + name = string + url = string + comment = optional(string, "Managed by Terraform") + read_only = optional(bool, false) + })) + description = "External locations to create. Each location has managed SQS file events enabled when enable_file_events is true." + + validation { + condition = length(var.external_locations) > 0 + error_message = "At least one external location is required." + } +} + +variable "enable_file_events" { + type = bool + description = "Enable automatic managed file events (SNS topic + SQS queue + S3 bucket notification, csms-* prefixed) on every external location. Adds the required sns/sqs/bucket-notification permissions to the IAM policy." + default = true +} + +variable "create_storage_credential" { + type = bool + description = "When true, create a storage credential backed by the IAM role this module manages. When false, reuse existing_credential_name (the IAM role/policy are still managed)." + default = true +} + +variable "existing_credential_name" { + type = string + description = "Name of an existing storage credential to reference on the external locations when create_storage_credential is false." + default = "" + + validation { + condition = var.create_storage_credential || var.existing_credential_name != "" + error_message = "existing_credential_name must be set when create_storage_credential is false." + } +} + +variable "storage_credential_name" { + type = string + description = "Name for the created storage credential. Defaults to \"-storage-credential\"." + default = "" +} + +variable "storage_credential_comment" { + type = string + description = "Comment applied to the created storage credential." + default = "Storage credential for external locations with file events. Managed by Terraform." +} + +variable "iam_role_name" { + type = string + description = "Name of the IAM role used by the storage credential. Defaults to \"-uc\"." + default = "" +} + +variable "iam_policy_name" { + type = string + description = "Name of the IAM policy attached to the role. Defaults to \"-policy\"." + default = "" +} + +variable "iam_propagation_delay" { + type = string + description = "Delay to wait after creating the IAM role/policy before creating the storage credential and external locations, so IAM changes propagate (avoids \"non self-assuming\" / 403 validation errors). Set to \"\" to disable the wait (e.g. when the role already exists)." + default = "60s" +} + +variable "force_destroy" { + type = bool + description = "Force destroy the storage credential and external locations even if dependents exist." + default = false +} + +variable "credential_grants" { + type = list(object({ + principal = string + privileges = list(string) + })) + description = "UC grants applied to the storage credential. Defaults to empty (owner-only)." + default = [] +} + +variable "location_grants" { + type = list(object({ + principal = string + privileges = list(string) + })) + description = <<-EOT + UC grants applied to every external location. + Recommended privileges for data engineers: BROWSE, READ_FILES, WRITE_FILES, + CREATE_EXTERNAL_TABLE, CREATE_EXTERNAL_VOLUME. + EOT + default = [] +} + +variable "tags" { + type = map(string) + description = "Tags applied to the IAM role and policy." + default = {} +} diff --git a/modules/aws-uc-external-location-file-events/versions.tf b/modules/aws-uc-external-location-file-events/versions.tf new file mode 100644 index 00000000..23eeccad --- /dev/null +++ b/modules/aws-uc-external-location-file-events/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + databricks = { + source = "databricks/databricks" + version = ">= 1.81.1" + } + time = { + source = "hashicorp/time" + version = ">= 0.9.0" + } + } +}