diff --git a/.gitignore b/.gitignore index a248d4ca..4f282ee0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ *.tfstate.lock. *.terraform.lock.hcl +# Terraform plan files +*.plan + # logs *.log @@ -22,6 +25,8 @@ # Ignored Terraform files *gitignore*.tf +terraform.tfvars +!terraform.tfvars.example # Ignore Mac .DS_Store files .DS_Store diff --git a/README.md b/README.md index 1c66db11..ac638e34 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ The folder `examples` contains the following Terraform implementation examples : | Azure | [adb-uc](examples/adb-uc/) | ADB Unity Catalog Process | | Azure | [adb-unity-catalog-basic-demo](examples/adb-unity-catalog-basic-demo/) | ADB Unity Catalog end-to-end demo including UC metastore setup, Users/groups sync from AAD to databricks account, UC Catalog, External locations, Schemas, & Access Grants | | Azure | [adb-overwatch](examples/adb-overwatch/) | Overwatch multi-workspace deployment on Azure | +| Azure | [adb-coding-assistants-cluster](examples/adb-coding-assistants-cluster/) | Databricks cluster with Claude Code CLI for AI-assisted development | | AWS | [aws-workspace-basic](examples/aws-workspace-basic/) | Provisioning AWS Databricks E2 | | AWS | [aws-workspace-with-firewall](examples/aws-workspace-with-firewall/) | Provisioning AWS Databricks E2 with an AWS Firewall | | AWS | [aws-exfiltration-protection](examples/aws-exfiltration-protection/) | An implementation of [Data Exfiltration Protection on AWS](https://www.databricks.com/blog/2021/02/02/data-exfiltration-protection-with-databricks-on-aws.html) | @@ -82,12 +83,14 @@ The folder `modules` contains the following Terraform modules : | Azure | [adb-overwatch-main-ws](modules/adb-overwatch-main-ws/) | Main Overwatch workspace deployment | | Azure | [adb-overwatch-ws-to-monitor](modules/adb-overwatch-ws-to-monitor/) | Overwatch deployment on the Azure workspace to monitor | | Azure | [adb-overwatch-analysis](modules/adb-overwatch-analysis/) | Overwatch analysis notebooks deployment on Azure | +| Azure | [adb-coding-assistants-cluster](modules/adb-coding-assistants-cluster/) | Databricks cluster with Claude Code CLI for AI-assisted development | | AWS | [aws-workspace-basic](modules/aws-workspace-basic/) | Provisioning AWS Databricks E2 | | AWS | [aws-databricks-base-infra](modules/aws-databricks-base-infra/) | Provisioning AWS Infrastructure to be used for the deployment of a Databricks E2 workspace | | AWS | [aws-databricks-unity-catalog](modules/aws-databricks-unity-catalog/) | Provisioning the AWS Infrastructure and setting up the metastore for Databricks Unity Catalog | | AWS | [aws-databricks-workspace](modules/aws-databricks-workspace/) | Provisioning AWS Databricks E2 Workspace using pre-created AWS Infra | | AWS | [aws-workspace-with-firewall](modules/aws-workspace-with-firewall/) | Provisioning AWS Databricks E2 with an AWS Firewall | | AWS | [aws-exfiltration-protection](modules/aws-exfiltration-protection/) | An implementation of [Data Exfiltration Protection on AWS](https://www.databricks.com/blog/2021/02/02/data-exfiltration-protection-with-databricks-on-aws.html) | +| AWS | [aws-uc-external-location-file-events](modules/aws-uc-external-location-file-events/) | Unity Catalog storage credential (self-assuming IAM role) and external location(s) with automatic managed SQS file events | | AWS | aws-workspace-with-private-link | Coming soon | | GCP | [gcp-sa-provisionning](modules/gcp-sa-provisionning/) | Provisions the identity (SA) with the correct permissions | | GCP | [gcp-workspace-basic](modules/gcp-workspace-basic/) | Provisions a workspace with managed VPC | diff --git a/examples/adb-coding-assistants-cluster/Makefile b/examples/adb-coding-assistants-cluster/Makefile new file mode 100644 index 00000000..653039d8 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/Makefile @@ -0,0 +1,7 @@ +.PHONY: docs test_docs + +docs: + terraform-docs -c ../../.terraform-docs.yml . + +test_docs: + terraform-docs -c ../../.terraform-docs.yml --output-check . diff --git a/examples/adb-coding-assistants-cluster/README.md b/examples/adb-coding-assistants-cluster/README.md new file mode 100644 index 00000000..6dfaa8d7 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/README.md @@ -0,0 +1,374 @@ +# Provisioning Databricks Cluster with Claude Code CLI + +This example uses the [adb-coding-assistants-cluster](../../modules/adb-coding-assistants-cluster) module. + +This template provides an example deployment of a Databricks cluster pre-configured with Claude Code CLI for AI-assisted development directly on the cluster. + +## What Gets Deployed + +* Unity Catalog Volume for init script storage +* Databricks cluster with Claude Code CLI auto-installed on startup +* MLflow experiment for tracing Claude Code sessions +* Bash helper functions for easy usage + +## How to use + +> **Note** +> A detailed module README with full configuration options can be found in [modules/adb-coding-assistants-cluster](../../modules/adb-coding-assistants-cluster) + +1. Reference this module using one of the different [module source types](https://developer.hashicorp.com/terraform/language/modules/sources) +2. Copy `terraform.tfvars.example` to `terraform.tfvars` +3. Update `terraform.tfvars` with your values: + - `databricks_resource_id`: Your Azure Databricks workspace resource ID + - `cluster_name`: Name for your cluster + - `catalog_name`: Unity Catalog name to use +4. (Optional) Customize cluster configuration in `terraform.tfvars` (node type, autoscaling, etc.) +5. (Optional) Configure your [remote backend](https://developer.hashicorp.com/terraform/language/settings/backends/azurerm) +6. Run `terraform init` to initialize terraform and get provider ready +7. Run `terraform plan` to review the resources that will be created +8. Run `terraform apply` to create the resources + +## Prerequisites + +- Databricks workspace with Unity Catalog enabled +- Unity Catalog with an existing catalog and schema +- **Unity Catalog metastore must have a root storage credential configured** (required for volumes) +- Permission to create clusters +- (For Azure) Authenticated via `az login` or environment variables +- Databricks Runtime 14.3 LTS or higher recommended + +> **Note**: If you encounter an error about missing root storage credential, you need to configure the metastore's root storage credential first. See [Databricks documentation](https://docs.databricks.com/api-explorer/workspace/metastores/update) for details. + +## Post-Deployment + +After the cluster starts, you can connect via SSH to use Claude Code and other development tools. + +### 1. Configure SSH Tunnel + +Use the Databricks CLI to set up SSH access to your new cluster: + +```bash +# Authenticate if needed +databricks auth login --host https://your-workspace-url.cloud.databricks.com + +# Set up SSH config (replace 'claude-dev' with your preferred alias) +databricks ssh setup --name claude-dev +# Select your cluster from the list when prompted +``` + +This creates an entry in your `~/.ssh/config` file. + +### 2. Connect via VSCode or Cursor + +1. Install the **Remote - SSH** extension in VSCode or Cursor. +2. Open the Command Palette (`Cmd+Shift+P` / `Ctrl+Shift+P`). +3. Select **Remote-SSH: Connect to Host**. +4. Choose `claude-dev` (or the alias you created). +5. Select **Linux** as the platform. +6. Once connected, open your persistent workspace folder: `/Workspace/Users//`. + +> **Important: Work Storage Location** +> ⚠️ **DO NOT use Databricks Repos (`/Repos/...`) for active development work.** Repos folders can be unreliable for persistent storage and may lose uncommitted changes during cluster restarts or sync operations. +> +> ✅ **Use `/Workspace/Users//` instead.** This location provides reliable persistent storage. You can use regular git commands to manage version control (see "Using Git in /Workspace" section below). + +### 3. Launch Claude Code + +Open the terminal in your remote VSCode/Cursor session and run: + +```bash +# 1. Load environment variables and helpers +source ~/.bashrc + +# 2. Enable MLflow tracing (optional but recommended) +claude-tracing-enable + +# 3. Start Claude Code +claude +``` + +**First-time setup tips:** +- Claude will ask for file permissions; use `Shift+Tab` to auto-allow edits in the current directory. +- If you need to refresh credentials, run `claude-refresh-token`. + +### 4. Remote Web App Development (Port Forwarding) + +VSCode and Cursor automatically forward ports. For example, to run a Streamlit app: + +1. Create `app.py`: + ```python + import streamlit as st + st.title("Databricks Remote App") + st.write("Running on cluster!") + ``` +2. Run it: + ```bash + streamlit run app.py --server.port 8501 + ``` +3. Click "Open in Browser" in the popup notification to view it at `localhost:8501`. + +### 5. Using the Databricks Python Interpreter + +You don't need to configure a virtual environment. Databricks manages it for you. + +1. In the remote terminal, find the python path: + ```bash + echo $DATABRICKS_VIRTUAL_ENV + # Output example: /local_disk0/.ephemeral_nfs/envs/pythonEnv-xxxx/bin/python + ``` +2. In VSCode/Cursor, open the Command Palette and select **Python: Select Interpreter**. +3. Paste the path from above. + +### 6. Persistent Sessions with tmux + +To keep your agent running even if you disconnect: + +```bash +# Start a new session +tmux new -s claude-session + +# Detach (Ctrl+B, then D) +# Reattach later +tmux attach -t claude-session +``` + +This allows you to leave long-running tasks (like "Build a data pipeline") executing on the cluster while you are offline. + +### 7. Using Git in /Workspace + +Since `/Workspace` doesn't have native Repos integration, use standard git commands: + +```bash +# Navigate to your workspace directory +cd /Workspace/Users// + +# Option 1: Clone an existing repository +git clone https://github.com/your-org/your-repo.git +cd your-repo + +# Option 2: Initialize a new repository +mkdir my-project && cd my-project +git init +git remote add origin https://github.com/your-org/your-repo.git + +# Configure git (first time only) +git config user.name "Your Name" +git config user.email "your.email@company.com" + +# Regular git workflow +git add . +git commit -m "Your commit message" +git push origin main +``` + +**Git Authentication Options:** + +1. **Personal Access Token (PAT)** - Recommended: + ```bash + # GitHub: Create at https://github.com/settings/tokens + # Use token as password when prompted + git clone https://github.com/your-org/repo.git + ``` + +2. **SSH Keys**: + ```bash + # Generate SSH key on the cluster + ssh-keygen -t ed25519 -C "your.email@company.com" + + # Add to GitHub: Copy output and add at https://github.com/settings/keys + cat ~/.ssh/id_ed25519.pub + + # Clone using SSH + git clone git@github.com:your-org/repo.git + ``` + +3. **Git Credential Manager**: + ```bash + # Store credentials to avoid repeated prompts + git config --global credential.helper store + ``` + +## Helper Commands + +### Claude CLI Commands + +| Command | Purpose | +|---------|---------| +| `check-claude` | Verify Claude CLI installation and configuration | +| `claude-debug` | Show detailed Claude configuration | +| `claude-refresh-token` | Regenerate Claude settings from environment | +| `claude-token-status` | Check token freshness and auto-refresh status | +| `claude-tracing-enable` | Enable MLflow tracing for Claude sessions | +| `claude-tracing-status` | Check tracing status | +| `claude-tracing-disable` | Disable tracing | + +### Git Workspace Commands + +| Command | Purpose | +|---------|---------| +| `git-workspace-init` | Interactive setup for git in /Workspace (clone or init) | +| `git-workspace-check` | Verify location and check for uncommitted/unpushed changes | +| `git-workspace-setup-auth` | Configure git authentication (PAT, SSH, or credential helper) | + +These helpers warn you if working in `/Repos` and ensure your work is backed up in git. + +### VS Code/Cursor Remote Commands + +| Command | Purpose | +|---------|---------| +| `claude-vscode-setup` | Show Remote SSH setup instructions | +| `claude-vscode-env` | Get Python interpreter path for IDE | +| `claude-vscode-check` | Verify Remote SSH configuration | +| `claude-vscode-config` | Generate settings.json snippet | + +## Offline Installation + +For air-gapped or restricted network environments, use the separate offline module: [`adb-coding-assistants-cluster-offline`](../../modules/adb-coding-assistants-cluster-offline/README.md). See the [Offline Installation Guide](../../modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md) for detailed instructions. + +## Configuration Examples + +### Single-Node Development Cluster + +```hcl +cluster_mode = "SINGLE_NODE" +num_workers = 0 +node_type_id = "Standard_D8pds_v6" +``` + +### Autoscaling Production Cluster + +```hcl +cluster_mode = "STANDARD" +num_workers = null # Enable autoscaling +min_workers = 2 +max_workers = 8 +node_type_id = "Standard_D8pds_v6" +``` + +## Authentication + +This example uses Databricks unified authentication. Authentication can be provided via: + +1. **Azure CLI** (recommended for local development): + ```bash + az login + terraform apply + ``` + +2. **Environment Variables** (recommended for CI/CD): + ```bash + export DATABRICKS_HOST="https://adb-xxx.azuredatabricks.net" + export DATABRICKS_TOKEN="dapi..." + terraform apply + ``` + +3. **Configuration Profile**: + ```bash + export DATABRICKS_CONFIG_PROFILE="my-profile" + terraform apply + ``` + +For more details on authentication, see the [Databricks unified authentication documentation](https://docs.databricks.com/dev-tools/auth/unified-auth.html). + +## Troubleshooting + +### Init Script Fails + +Check cluster event logs in the Databricks UI under **Compute** → **Your Cluster** → **Event Log**. + +Common issues: +- Network connectivity to download packages +- Unity Catalog volume permissions +- Insufficient cluster permissions + +### Claude Not Found After Login + +```bash +# Reload bashrc +source ~/.bashrc + +# Verify PATH +check-claude +``` + +### Authentication Issues + +```bash +# Check environment variables +check-claude + +# Regenerate configuration +claude-refresh-token +``` + +## Additional Resources + +- [Module Documentation](../../modules/adb-coding-assistants-cluster/README.md) +- [Offline Module Documentation](../../modules/adb-coding-assistants-cluster-offline/README.md) +- [Offline Installation Guide](../../modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md) +- [Scripts Documentation](../../modules/adb-coding-assistants-cluster/scripts/README.md) +- [Databricks Init Scripts Documentation](https://docs.databricks.com/clusters/init-scripts.html) +- [Unity Catalog Volumes Documentation](https://docs.databricks.com/data-governance/unity-catalog/volumes.html) + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [azurerm](#requirement\_azurerm) | >=4.31.0 | +| [databricks](#requirement\_databricks) | >=1.81.1 | + +## Providers + +| Name | Version | +|------|---------| +| [azurerm](#provider\_azurerm) | 4.57.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [claude\_cluster](#module\_claude\_cluster) | ../../modules/adb-coding-assistants-cluster | n/a | + +## Resources + +| Name | Type | +|------|------| +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_databricks_workspace.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/databricks_workspace) | data source | +| [azurerm_resource_group.this](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [catalog\_name](#input\_catalog\_name) | Unity Catalog name for the volume | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Name of the Databricks cluster | `string` | n/a | yes | +| [databricks\_resource\_id](#input\_databricks\_resource\_id) | The Azure resource ID for the Databricks workspace. Format: /subscriptions/{subscription-id}/resourceGroups/{resource-group}/providers/Microsoft.Databricks/workspaces/{workspace-name} | `string` | n/a | yes | +| [autotermination\_minutes](#input\_autotermination\_minutes) | Minutes of inactivity before cluster auto-terminates | `number` | `30` | no | +| [cluster\_mode](#input\_cluster\_mode) | Cluster mode: STANDARD or SINGLE\_NODE | `string` | `"STANDARD"` | no | +| [init\_script\_source\_path](#input\_init\_script\_source\_path) | Local path to the init script | `string` | `null` | no | +| [max\_workers](#input\_max\_workers) | Maximum number of workers for autoscaling | `number` | `3` | no | +| [min\_workers](#input\_min\_workers) | Minimum number of workers for autoscaling | `number` | `1` | no | +| [mlflow\_experiment\_name](#input\_mlflow\_experiment\_name) | MLflow experiment name for Claude Code tracing | `string` | `"/Workspace/Shared/claude-code-tracing"` | no | +| [node\_type\_id](#input\_node\_type\_id) | Node type for the cluster. Default is Standard_D8pds_v6 (modern, premium SSD + local NVMe). If unavailable in your region, consider Standard_DS13_v2 as fallback. | `string` | `"Standard_D8pds_v6"` | no | +| [num\_workers](#input\_num\_workers) | Number of worker nodes (null for autoscaling) | `number` | `null` | no | +| [schema\_name](#input\_schema\_name) | Schema name for the volume | `string` | `"default"` | no | +| [spark\_version](#input\_spark\_version) | Databricks Runtime version | `string` | `"17.3.x-cpu-ml-scala2.13"` | no | +| [tags](#input\_tags) | Custom tags for the cluster | `map(string)` | 
{
  "Environment": "dev",
  "Purpose": "coding-assistants"
}
| no | +| [volume\_name](#input\_volume\_name) | Volume name to store init scripts | `string` | `"coding_assistants"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | The ID of the created cluster | +| [cluster\_name](#output\_cluster\_name) | Name of the created cluster | +| [cluster\_url](#output\_cluster\_url) | URL to access the cluster in Databricks UI | +| [init\_script\_path](#output\_init\_script\_path) | Path to the init script in the volume | +| [mlflow\_experiment\_name](#output\_mlflow\_experiment\_name) | MLflow experiment name for tracing | +| [setup\_instructions](#output\_setup\_instructions) | Instructions for using the cluster | +| [volume\_full\_name](#output\_volume\_full\_name) | Full name of the volume | +| [volume\_path](#output\_volume\_path) | Path to the volume containing init scripts | + diff --git a/examples/adb-coding-assistants-cluster/main.tf b/examples/adb-coding-assistants-cluster/main.tf new file mode 100644 index 00000000..cd2519e6 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/main.tf @@ -0,0 +1,21 @@ +# Cluster with Claude Code CLI coding assistant +# Provider configuration is in providers.tf +module "claude_cluster" { + source = "../../modules/adb-coding-assistants-cluster" + + cluster_name = var.cluster_name + catalog_name = var.catalog_name + schema_name = var.schema_name + volume_name = var.volume_name + init_script_source_path = var.init_script_source_path + spark_version = var.spark_version + node_type_id = var.node_type_id + autotermination_minutes = var.autotermination_minutes + num_workers = var.num_workers + min_workers = var.min_workers + max_workers = var.max_workers + mlflow_experiment_name = var.mlflow_experiment_name + cluster_mode = var.cluster_mode + tags = var.tags +} + diff --git a/examples/adb-coding-assistants-cluster/outputs.tf b/examples/adb-coding-assistants-cluster/outputs.tf new file mode 100644 index 00000000..6c3cce86 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/outputs.tf @@ -0,0 +1,58 @@ +output "cluster_id" { + description = "The ID of the created cluster" + value = module.claude_cluster.cluster_id +} + +output "cluster_url" { + description = "URL to access the cluster in Databricks UI" + value = module.claude_cluster.cluster_url +} + +output "cluster_name" { + description = "Name of the created cluster" + value = module.claude_cluster.cluster_name +} + +output "volume_path" { + description = "Path to the volume containing init scripts" + value = module.claude_cluster.volume_path +} + +output "volume_full_name" { + description = "Full name of the volume" + value = module.claude_cluster.volume_full_name +} + +output "init_script_path" { + description = "Path to the init script in the volume" + value = module.claude_cluster.init_script_path +} + +output "mlflow_experiment_name" { + description = "MLflow experiment name for tracing" + value = module.claude_cluster.mlflow_experiment_name +} + +output "setup_instructions" { + description = "Instructions for using the cluster" + value = <<-EOT + Cluster deployed successfully! + + 1. Access cluster: ${module.claude_cluster.cluster_url} + 2. Wait for cluster to start (init script runs automatically) + 3. Open a notebook or terminal + 4. Run: source ~/.bashrc + 5. Verify: check-claude + 6. Start using: claude "your question" + + MLflow Experiment: ${module.claude_cluster.mlflow_experiment_name} + + Helper commands: + - check-claude: Verify installation status + - claude-debug: Show configuration details + - claude-refresh-token: Update authentication + - claude-tracing-enable: Enable MLflow tracing + - claude-tracing-status: Check tracing status + - claude-tracing-disable: Disable tracing + EOT +} diff --git a/examples/adb-coding-assistants-cluster/providers.tf b/examples/adb-coding-assistants-cluster/providers.tf new file mode 100644 index 00000000..26c208e7 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/providers.tf @@ -0,0 +1,91 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">=4.31.0" + } + databricks = { + source = "databricks/databricks" + version = ">=1.81.1" + } + external = { + source = "hashicorp/external" + version = ">=2.3.0" + } + } +} + +# Determine authentication approach based on variables provided +locals { + # Use profile-based auth if profile is specified + use_profile_auth = var.databricks_profile != null + + # For Azure resource ID approach + resource_regex = var.databricks_resource_id != null ? "(?i)subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft.Databricks/workspaces/(.+)" : "" + subscription_id_from_resource = var.databricks_resource_id != null ? regex(local.resource_regex, var.databricks_resource_id)[0] : null + resource_group = var.databricks_resource_id != null ? regex(local.resource_regex, var.databricks_resource_id)[1] : null + databricks_workspace_name = var.databricks_resource_id != null ? regex(local.resource_regex, var.databricks_resource_id)[2] : null +} + +# Get Azure subscription ID from Azure CLI or environment variable when not provided via resource ID +# This is needed for the Azure provider even when using profile-based Databricks auth +data "external" "azure_subscription" { + count = local.subscription_id_from_resource == null ? 1 : 0 + program = ["bash", "-c", "SUBSCRIPTION_ID=$(az account show --query id -o tsv 2>/dev/null || echo $${ARM_SUBSCRIPTION_ID:-}); echo \"{\\\"id\\\":\\\"$${SUBSCRIPTION_ID:-}\\\"}\""] +} + +locals { + # Use subscription ID from resource ID, or from Azure CLI/environment, or null (provider will try to auto-detect) + subscription_id = coalesce( + local.subscription_id_from_resource, + try(data.external.azure_subscription[0].result.id != "" ? data.external.azure_subscription[0].result.id : null, null) + ) +} + +# Data source to get current Azure client configuration (only for Azure resource ID approach) +data "azurerm_client_config" "current" { + count = local.use_profile_auth ? 0 : 1 +} + +# Data source to get the resource group (only for Azure resource ID approach) +data "azurerm_resource_group" "this" { + count = local.use_profile_auth ? 0 : 1 + name = local.resource_group +} + +# Configure the Azure Provider +# When using profile-based auth, subscription_id is not needed (provider will auto-detect if Azure CLI is configured) +# When using Azure resource ID approach, subscription_id is extracted from the resource ID +provider "azurerm" { + subscription_id = local.subscription_id + features {} + skip_provider_registration = local.use_profile_auth + + # Allow provider to work without explicit subscription_id when using profile auth + # It will attempt to auto-detect from Azure CLI or environment variables +} + +# Data source to get the Databricks workspace (only for Azure resource ID approach) +data "azurerm_databricks_workspace" "this" { + count = local.use_profile_auth ? 0 : 1 + name = local.databricks_workspace_name + resource_group_name = local.resource_group +} + +# Configure the Databricks Provider +# Two authentication approaches supported: +# +# 1. Profile-based (Recommended - Simple and cloud-agnostic): +# Set databricks_profile variable to your ~/.databrickscfg profile name +# Example: databricks_profile = "dok" +# +# 2. Azure resource ID (Azure-specific): +# Set databricks_resource_id to your Azure Databricks workspace resource ID +# Requires Azure CLI authentication (az login) +# +# See: https://docs.databricks.com/dev-tools/auth/unified-auth.html +provider "databricks" { + profile = var.databricks_profile + host = local.use_profile_auth ? null : data.azurerm_databricks_workspace.this[0].workspace_url +} + diff --git a/examples/adb-coding-assistants-cluster/terraform.tfvars.example b/examples/adb-coding-assistants-cluster/terraform.tfvars.example new file mode 100644 index 00000000..32296df2 --- /dev/null +++ b/examples/adb-coding-assistants-cluster/terraform.tfvars.example @@ -0,0 +1,88 @@ +# Example terraform.tfvars file for Claude Code CLI Cluster +# Copy this to terraform.tfvars and customize for your environment + +#============================================================================= +# AUTHENTICATION - Choose ONE approach +#============================================================================= + +# OPTION 1: Profile-based (Recommended - Simple and cloud-agnostic) +# Uses your ~/.databrickscfg profile +databricks_profile = "my-profile" # Replace with your profile name from ~/.databrickscfg + +# OPTION 2: Azure Resource ID (Azure-specific) +# Comment out databricks_profile above and uncomment below to use Azure resource ID +# databricks_resource_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/my-rg/providers/Microsoft.Databricks/workspaces/my-workspace" + +#============================================================================= +# REQUIRED VARIABLES +#============================================================================= + +cluster_name = "claude-coding-assistant" +catalog_name = "main" + +#============================================================================= +# OPTIONAL VARIABLES +#============================================================================= + +# Unity Catalog configuration +schema_name = "default" +volume_name = "coding_assistants" + +# Cluster configuration +spark_version = "17.3.x-cpu-ml-scala2.13" +node_type_id = "Standard_D8pds_v6" # Azure: Standard_D8pds_v6 (8 vCPU, 32 GB RAM, Premium SSD + local NVMe). Fallback: Standard_DS13_v2 if unavailable in region +autotermination_minutes = 30 + +# Cluster mode options: +# - "SINGLE_NODE": Cost-effective for individual development (recommended) +# - "STANDARD": Multi-node for team environments +cluster_mode = "SINGLE_NODE" + +# Worker configuration (ignored if cluster_mode = "SINGLE_NODE") +num_workers = 0 # Set to null to enable autoscaling, or a specific number + +# Autoscaling configuration (only used if num_workers = null) +min_workers = 1 +max_workers = 3 + +# MLflow experiment for Claude Code tracing +mlflow_experiment_name = "/Workspace/Shared/claude-code-tracing" + +# Optional: Custom init script path (defaults to bundled script) +# init_script_source_path = "./custom-install-claude.sh" + +# Custom tags +tags = { + Environment = "development" + Purpose = "ai-coding" + Owner = "data-engineering" + CostCenter = "engineering" +} + +#============================================================================= +# CLOUD-SPECIFIC NODE TYPES REFERENCE +#============================================================================= + +# Azure VM Types (Premium SSD): +# Modern Dpdsv6-series (Cobalt 100 processor, Premium SSD + local NVMe): +# - Standard_D4pds_v6 (4 cores, 16 GB RAM) - Cost-effective +# - Standard_D8pds_v6 (8 cores, 32 GB RAM) - Recommended default (modern) +# - Standard_D16pds_v6 (16 cores, 64 GB RAM) - For larger workloads +# Note: Dpdsv6-series may have limited regional availability +# +# DS-series (Premium SSD, widely available): +# - Standard_DS3_v2 (4 cores, 14 GB RAM) - Cost-effective for development +# - Standard_DS4_v2 (8 cores, 28 GB RAM) - Good for medium workloads +# - Standard_DS13_v2 (8 cores, 56 GB RAM) - Good fallback if Dpdsv6 unavailable +# - Standard_DS5_v2 (16 cores, 56 GB RAM) - More CPU, same RAM as DS13_v2 +# - Standard_DS14_v2 (16 cores, 112 GB RAM) - For large-scale workloads +# +# AWS: +# - i3.xlarge (4 cores, 30.5 GB RAM) - Recommended for single-node +# - i3.2xlarge (8 cores, 61 GB RAM) - For larger workloads +# - r5.xlarge (4 cores, 32 GB RAM) - Memory-optimized +# +# GCP: +# - n1-highmem-4 (4 cores, 26 GB RAM) - Recommended for single-node +# - n1-highmem-8 (8 cores, 52 GB RAM) - For larger workloads +# - n2-standard-4 (4 cores, 16 GB RAM) - Cost-optimized diff --git a/examples/adb-coding-assistants-cluster/variables.tf b/examples/adb-coding-assistants-cluster/variables.tf new file mode 100644 index 00000000..4dc0f3ac --- /dev/null +++ b/examples/adb-coding-assistants-cluster/variables.tf @@ -0,0 +1,106 @@ +variable "databricks_profile" { + description = "Databricks CLI profile name from ~/.databrickscfg (recommended for simple, cloud-agnostic authentication). If set, databricks_resource_id is ignored." + type = string + default = null +} + +variable "databricks_resource_id" { + description = "The Azure resource ID for the Databricks workspace (Azure-specific approach). Format: /subscriptions/{subscription-id}/resourceGroups/{resource-group}/providers/Microsoft.Databricks/workspaces/{workspace-name}. Only used if databricks_profile is not set." + type = string + default = null + + validation { + condition = var.databricks_profile != null || var.databricks_resource_id != null + error_message = "Either databricks_profile or databricks_resource_id must be set. Recommended: use databricks_profile for simpler configuration." + } +} + +variable "cluster_name" { + description = "Name of the Databricks cluster" + type = string +} + +variable "catalog_name" { + description = "Unity Catalog name for the volume" + type = string +} + +variable "schema_name" { + description = "Schema name for the volume" + type = string + default = "default" +} + +variable "volume_name" { + description = "Volume name to store init scripts" + type = string + default = "coding_assistants" +} + +variable "init_script_source_path" { + description = "Local path to the init script" + type = string + default = null +} + +variable "spark_version" { + description = "Databricks Runtime version" + type = string + default = "17.3.x-cpu-ml-scala2.13" +} + +variable "node_type_id" { + description = "Node type for the cluster. Default is Standard_D8pds_v6 (modern, premium SSD + local NVMe). If unavailable in your region, consider Standard_DS13_v2 as fallback." + type = string + default = "Standard_D8pds_v6" +} + +variable "autotermination_minutes" { + description = "Minutes of inactivity before cluster auto-terminates" + type = number + default = 30 +} + +variable "num_workers" { + description = "Number of worker nodes (null for autoscaling)" + type = number + default = null +} + +variable "min_workers" { + description = "Minimum number of workers for autoscaling" + type = number + default = 1 +} + +variable "max_workers" { + description = "Maximum number of workers for autoscaling" + type = number + default = 3 +} + +variable "mlflow_experiment_name" { + description = "MLflow experiment name for Claude Code tracing" + type = string + default = "/Workspace/Shared/claude-code-tracing" +} + +variable "cluster_mode" { + description = "Cluster mode: STANDARD or SINGLE_NODE" + type = string + default = "STANDARD" + + validation { + condition = contains(["STANDARD", "SINGLE_NODE"], var.cluster_mode) + error_message = "cluster_mode must be either STANDARD or SINGLE_NODE" + } +} + +variable "tags" { + description = "Custom tags for the cluster" + type = map(string) + default = { + Environment = "dev" + Purpose = "coding-assistants" + } +} diff --git a/examples/adb-coding-assistants-cluster/versions.tf b/examples/adb-coding-assistants-cluster/versions.tf new file mode 100644 index 00000000..7117131f --- /dev/null +++ b/examples/adb-coding-assistants-cluster/versions.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 1.0" +} diff --git a/examples/aws-uc-external-location-file-events/README.md b/examples/aws-uc-external-location-file-events/README.md new file mode 100644 index 00000000..262555e2 --- /dev/null +++ b/examples/aws-uc-external-location-file-events/README.md @@ -0,0 +1,88 @@ +# Example: aws-uc-external-location-file-events + +Creates an S3 bucket and uses the +[`aws-uc-external-location-file-events`](../../modules/aws-uc-external-location-file-events) +module to create a Unity Catalog storage credential (backed by a self-assuming IAM role) and an +external location with **automatic managed file events** (SNS + SQS, `csms-*` prefixed). + +## Prerequisites + +- A Unity Catalog–enabled Databricks workspace whose assigned metastore is the target metastore. +- AWS credentials for the account where the bucket/IAM role live (profile or env vars). +- Databricks auth for that workspace (CLI profile or `DATABRICKS_*` env vars). +- Permission to create IAM roles/policies and UC storage credentials + external locations. + +## Usage + +```bash +cp terraform.tfvars.example terraform.tfvars +# edit terraform.tfvars + +terraform init +terraform plan +terraform apply +``` + +After apply, verify file events with the Unity Catalog storage-credential validation API: + +```bash +databricks api post /api/2.1/unity-catalog/validate-storage-credentials \ + --json '{"storage_credential_name":"","external_location_name":""}' +``` + +`READ_MESSAGE` transitions from `SKIP` ("being provisioned") to `PASS` once Databricks +provisions the managed SNS topic + SQS queue and the S3 bucket notification. + + +## Requirements + +| Name | Version | +| ---- | ------- | +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | >= 5.0 | +| [databricks](#requirement\_databricks) | >= 1.81.1 | +| [time](#requirement\_time) | >= 0.9.0 | + +## Providers + +| Name | Version | +| ---- | ------- | +| [aws](#provider\_aws) | >= 5.0 | + +## Modules + +| Name | Source | Version | +| ---- | ------ | ------- | +| [uc\_external\_location\_file\_events](#module\_uc\_external\_location\_file\_events) | ../../modules/aws-uc-external-location-file-events | n/a | + +## Resources + +| Name | Type | +| ---- | ---- | +| [aws_s3_bucket.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket_public_access_block.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| [bucket\_name](#input\_bucket\_name) | Globally unique S3 bucket name to create for the external location | `string` | n/a | yes | +| [databricks\_account\_id](#input\_databricks\_account\_id) | Databricks account ID (used as the sts:ExternalId in the IAM role trust policy). | `string` | n/a | yes | +| [aws\_profile](#input\_aws\_profile) | AWS CLI profile for the account that owns the bucket. Leave empty to use the default credential chain / env vars. | `string` | `""` | no | +| [aws\_region](#input\_aws\_region) | AWS region for the S3 bucket | `string` | `"us-east-1"` | no | +| [databricks\_profile](#input\_databricks\_profile) | Databricks CLI profile for a workspace assigned to the target metastore. Leave empty to use DATABRICKS\_* env vars. | `string` | `""` | no | +| [external\_location\_name](#input\_external\_location\_name) | Name of the Unity Catalog external location | `string` | `"file_events_landing"` | no | +| [external\_location\_prefix](#input\_external\_location\_prefix) | Prefix within the bucket managed by the external location | `string` | `"landing"` | no | +| [grant\_principal](#input\_grant\_principal) | UC group or user to grant on the credential and external location. Leave empty to skip grants. | `string` | `""` | no | +| [name\_prefix](#input\_name\_prefix) | Prefix for UC object and IAM names | `string` | `"file-events-demo"` | no | + +## Outputs + +| Name | Description | +| ---- | ----------- | +| [bucket\_name](#output\_bucket\_name) | n/a | +| [external\_location\_ids](#output\_external\_location\_ids) | n/a | +| [external\_location\_urls](#output\_external\_location\_urls) | n/a | +| [iam\_role\_arn](#output\_iam\_role\_arn) | n/a | +| [storage\_credential\_name](#output\_storage\_credential\_name) | n/a | + diff --git a/examples/aws-uc-external-location-file-events/main.tf b/examples/aws-uc-external-location-file-events/main.tf new file mode 100644 index 00000000..3f25b628 --- /dev/null +++ b/examples/aws-uc-external-location-file-events/main.tf @@ -0,0 +1,43 @@ +# The caller owns the bucket; the module wires up IAM + Unity Catalog. +resource "aws_s3_bucket" "this" { + bucket = var.bucket_name + force_destroy = true +} + +resource "aws_s3_bucket_public_access_block" "this" { + bucket = aws_s3_bucket.this.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +module "uc_external_location_file_events" { + source = "../../modules/aws-uc-external-location-file-events" + + name_prefix = var.name_prefix + databricks_account_id = var.databricks_account_id + bucket_names = [aws_s3_bucket.this.id] + + external_locations = [ + { + name = var.external_location_name + url = "s3://${aws_s3_bucket.this.id}/${var.external_location_prefix}" + comment = "Landing zone with managed SQS file events" + } + ] + + credential_grants = var.grant_principal == "" ? [] : [ + { + principal = var.grant_principal + privileges = ["CREATE_EXTERNAL_LOCATION", "READ_FILES", "WRITE_FILES"] + } + ] + + location_grants = var.grant_principal == "" ? [] : [ + { + principal = var.grant_principal + privileges = ["BROWSE", "READ_FILES", "WRITE_FILES", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME"] + } + ] +} diff --git a/examples/aws-uc-external-location-file-events/outputs.tf b/examples/aws-uc-external-location-file-events/outputs.tf new file mode 100644 index 00000000..ea9ee8bb --- /dev/null +++ b/examples/aws-uc-external-location-file-events/outputs.tf @@ -0,0 +1,19 @@ +output "bucket_name" { + value = aws_s3_bucket.this.id +} + +output "iam_role_arn" { + value = module.uc_external_location_file_events.iam_role_arn +} + +output "storage_credential_name" { + value = module.uc_external_location_file_events.storage_credential_name +} + +output "external_location_ids" { + value = module.uc_external_location_file_events.external_location_ids +} + +output "external_location_urls" { + value = module.uc_external_location_file_events.external_location_urls +} diff --git a/examples/aws-uc-external-location-file-events/providers.tf b/examples/aws-uc-external-location-file-events/providers.tf new file mode 100644 index 00000000..4a143cec --- /dev/null +++ b/examples/aws-uc-external-location-file-events/providers.tf @@ -0,0 +1,11 @@ +provider "aws" { + region = var.aws_region + profile = var.aws_profile +} + +# Authenticate to an existing UC-enabled workspace whose assigned metastore is the +# one you want these objects in. Prefer `databricks auth login --host ` +# (a CLI profile) or environment variables over inline credentials. +provider "databricks" { + profile = var.databricks_profile +} diff --git a/examples/aws-uc-external-location-file-events/terraform.tfvars.example b/examples/aws-uc-external-location-file-events/terraform.tfvars.example new file mode 100644 index 00000000..acb4b48d --- /dev/null +++ b/examples/aws-uc-external-location-file-events/terraform.tfvars.example @@ -0,0 +1,12 @@ +aws_region = "ap-southeast-2" +aws_profile = "my-aws-profile" +databricks_profile = "my-workspace-profile" + +databricks_account_id = "00000000-0000-0000-0000-000000000000" + +name_prefix = "file-events-demo" +bucket_name = "my-globally-unique-uc-demo-bucket" +external_location_name = "file_events_landing" +external_location_prefix = "landing" + +grant_principal = "data-engineers" diff --git a/examples/aws-uc-external-location-file-events/variables.tf b/examples/aws-uc-external-location-file-events/variables.tf new file mode 100644 index 00000000..b94cac8d --- /dev/null +++ b/examples/aws-uc-external-location-file-events/variables.tf @@ -0,0 +1,51 @@ +variable "aws_region" { + type = string + description = "AWS region for the S3 bucket" + default = "us-east-1" +} + +variable "aws_profile" { + type = string + description = "AWS CLI profile for the account that owns the bucket. Leave empty to use the default credential chain / env vars." + default = "" +} + +variable "databricks_profile" { + type = string + description = "Databricks CLI profile for a workspace assigned to the target metastore. Leave empty to use DATABRICKS_* env vars." + default = "" +} + +variable "databricks_account_id" { + type = string + description = "Databricks account ID (used as the sts:ExternalId in the IAM role trust policy)." +} + +variable "name_prefix" { + type = string + description = "Prefix for UC object and IAM names" + default = "file-events-demo" +} + +variable "bucket_name" { + type = string + description = "Globally unique S3 bucket name to create for the external location" +} + +variable "external_location_name" { + type = string + description = "Name of the Unity Catalog external location" + default = "file_events_landing" +} + +variable "external_location_prefix" { + type = string + description = "Prefix within the bucket managed by the external location" + default = "landing" +} + +variable "grant_principal" { + type = string + description = "UC group or user to grant on the credential and external location. Leave empty to skip grants." + default = "" +} diff --git a/examples/aws-uc-external-location-file-events/versions.tf b/examples/aws-uc-external-location-file-events/versions.tf new file mode 100644 index 00000000..23eeccad --- /dev/null +++ b/examples/aws-uc-external-location-file-events/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + databricks = { + source = "databricks/databricks" + version = ">= 1.81.1" + } + time = { + source = "hashicorp/time" + version = ">= 0.9.0" + } + } +} diff --git a/modules/adb-coding-assistants-cluster-offline/README.md b/modules/adb-coding-assistants-cluster-offline/README.md new file mode 100644 index 00000000..356f743a --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/README.md @@ -0,0 +1,358 @@ +# Provisioning Databricks Cluster with Claude Code CLI (Offline Installation) + +This module deploys a Databricks cluster pre-configured with Claude Code CLI for AI-assisted development in **air-gapped or restricted network environments**. + +## Module content + +This module can be used to deploy the following: + +* Unity Catalog Volume for secure init script storage +* Init script with Claude Code CLI offline installation +* Databricks cluster with automatic AI coding assistant setup +* MLflow experiment configuration for tracing +* Helper bash functions for cluster users + +## Features + +- ✅ **Zero-configuration AI coding tools** on cluster startup +- ✅ **Unity Catalog Volumes** for secure script storage (Databricks recommended practice) +- ✅ **MLflow tracing** integration for Claude Code sessions +- ✅ **Flexible cluster configuration** (single-node or autoscaling) +- ✅ **Offline/air-gapped installation** - no internet access required + +## Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ Unity Catalog Volume │ +│ /Volumes//// │ +│ └── install-claude-offline.sh │ +└─────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────┐ +│ Databricks Cluster (on startup) │ +│ │ +│ 1. Executes init script from volume │ +│ 2. Installs Node.js, Claude CLI from local packages│ +│ 3. Configures bashrc with helper functions │ +│ 4. Auto-generates configs on user login │ +└─────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────┐ +│ User Login │ +│ │ +│ • DATABRICKS_TOKEN available from environment │ +│ • Configs auto-generate: │ +│ - ~/.claude/settings.json │ +│ • Commands ready: claude │ +└─────────────────────────────────────────────────────┘ +``` + +## Prerequisites + +- Databricks workspace with Unity Catalog enabled +- Databricks Runtime 13.3 LTS or higher (recommended for Unity Catalog volumes) +- Databricks Terraform provider >= 1.40.0 +- Unity Catalog with an existing catalog and schema +- **Offline packages** prepared and uploaded to DBFS (see [Offline Installation Guide](scripts/OFFLINE-INSTALLATION.md)) + +## Usage + +### Basic Example + +```hcl +module "coding_cluster_offline" { + source = "./modules/adb-coding-assistants-cluster-offline" + + cluster_name = "ai-dev-cluster-offline" + catalog_name = "main" + schema_name = "default" + + # Path to offline packages (defaults to /dbfs/init-scripts/offline-packages) + offline_packages_path = "/dbfs/init-scripts/offline-packages" +} +``` + +### Complete Example + +```hcl +module "coding_cluster_offline" { + source = "./modules/adb-coding-assistants-cluster-offline" + + # Cluster configuration + cluster_name = "ai-development-cluster-offline" + spark_version = "14.3.x-scala2.12" + node_type_id = "Standard_DS3_v2" + autotermination_minutes = 60 + + # Volume configuration + catalog_name = "main" + schema_name = "default" + volume_name = "coding_assistants_offline" + + # Offline packages path + offline_packages_path = "/dbfs/init-scripts/offline-packages" + + # MLflow configuration + mlflow_experiment_name = "/Workspace/Shared/claude-code-tracing" + + # Autoscaling + min_workers = 1 + max_workers = 5 + + # Tags + tags = { + Environment = "development" + Project = "ai-assisted-coding" + CostCenter = "engineering" + ManagedBy = "terraform" + } +} +``` + +## Preparing Offline Packages + +Before using this module, you must prepare offline packages: + +1. **Download dependencies** (on a machine with internet access): + ```bash + cd modules/adb-coding-assistants-cluster-offline/scripts + ./download-offline-dependencies.sh + ``` + +2. **Upload to Databricks**: + ```bash + databricks fs cp -r offline-packages/ dbfs:/init-scripts/offline-packages/ + ``` + +3. **Configure the module** with `offline_packages_path` pointing to the uploaded location. + +See [scripts/OFFLINE-INSTALLATION.md](scripts/OFFLINE-INSTALLATION.md) for detailed instructions. + +## Init Script Storage Best Practices + +According to [Databricks documentation](https://docs.databricks.com/aws/en/init-scripts/): + +> **Databricks Runtime 13.3 LTS and above with Unity Catalog** +> Store init scripts in Unity Catalog volumes. + +### Why Unity Catalog Volumes? + +1. **Governance**: Full Unity Catalog ACL support +2. **Security**: Identity-based access control +3. **Portability**: Works across AWS, Azure, and GCP +4. **Versioning**: Easy to manage and update scripts +5. **No DBFS**: Recommended alternative to legacy DBFS storage + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [databricks](#requirement\_databricks) | >= 1.40.0 | + +## Providers + +| Name | Version | +|------|---------| +| [databricks](#provider\_databricks) | >= 1.40.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [databricks_cluster.coding_assistants](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster) | resource | +| [databricks_file.init_script](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/file) | resource | +| [databricks_volume.init_scripts](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/volume) | resource | +| [databricks_current_user.me](https://registry.terraform.io/providers/databricks/databricks/latest/docs/data-sources/current_user) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [catalog\_name](#input\_catalog\_name) | Unity Catalog name for the volume | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Name of the Databricks cluster | `string` | n/a | yes | +| [autotermination\_minutes](#input\_autotermination\_minutes) | Minutes of inactivity before cluster auto-terminates | `number` | `30` | no | +| [cluster\_mode](#input\_cluster\_mode) | Cluster mode: STANDARD or SINGLE\_NODE | `string` | `"STANDARD"` | no | +| [init\_script\_source\_path](#input\_init\_script\_source\_path) | Local path to the init script | `string` | `null` | no | +| [max\_workers](#input\_max\_workers) | Maximum number of workers for autoscaling | `number` | `3` | no | +| [min\_workers](#input\_min\_workers) | Minimum number of workers for autoscaling | `number` | `1` | no | +| [mlflow\_experiment\_name](#input\_mlflow\_experiment\_name) | MLflow experiment name for Claude Code tracing | `string` | `"/Workspace/Shared/claude-code-tracing"` | no | +| [node\_type\_id](#input\_node\_type\_id) | Node type for the cluster | `string` | `"Standard_DS3_v2"` | no | +| [num\_workers](#input\_num\_workers) | Number of worker nodes (null for autoscaling) | `number` | `null` | no | +| [offline\_packages\_path](#input\_offline\_packages\_path) | Path to offline packages directory (e.g., /dbfs/init-scripts/offline-packages). If not set, defaults to /dbfs/init-scripts/offline-packages | `string` | `null` | no | +| [schema\_name](#input\_schema\_name) | Schema name for the volume | `string` | `"default"` | no | +| [spark\_version](#input\_spark\_version) | Databricks Runtime version | `string` | `"14.3.x-scala2.12"` | no | +| [tags](#input\_tags) | Custom tags for the cluster | `map(string)` | 
{
  "Environment": "dev",
  "Purpose": "coding-assistants-offline"
}
| no | +| [volume\_name](#input\_volume\_name) | Volume name to store init scripts | `string` | `"coding_assistants"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | The ID of the created cluster | +| [cluster\_name](#output\_cluster\_name) | Name of the created cluster | +| [cluster\_url](#output\_cluster\_url) | URL to access the cluster in Databricks UI | +| [init\_script\_path](#output\_init\_script\_path) | Path to the init script in the volume | +| [mlflow\_experiment\_name](#output\_mlflow\_experiment\_name) | MLflow experiment name for tracing | +| [offline\_packages\_path](#output\_offline\_packages\_path) | Path to offline packages directory | +| [volume\_full\_name](#output\_volume\_full\_name) | Full name of the volume | +| [volume\_path](#output\_volume\_path) | Path to the volume containing init scripts | + + +## Post-Deployment Usage + +### On the Cluster + +After the cluster starts, users can: + +```bash +# Check installation status +check-claude + +# Debug Claude configuration +claude-debug + +# Use Claude Code +claude "Analyze the customer churn data" + +# Enable MLflow tracing +claude-tracing-enable + +# Check tracing status +claude-tracing-status +``` + +### Helper Commands + +The init script installs these helper commands in `~/.bashrc`: + +| Command | Purpose | +|---------|---------| +| `check-claude` | Verify installation and configuration | +| `claude-debug` | Show detailed Claude CLI configuration | +| `claude-refresh-token` | Regenerate Claude settings | +| `claude-tracing-enable` | Enable MLflow tracing | +| `claude-tracing-status` | Check tracing status | +| `claude-tracing-disable` | Disable MLflow tracing | +| `claude-vscode-setup` | Show VS Code/Cursor Remote SSH setup guide | +| `claude-vscode-env` | Get Python virtual environment path | +| `claude-vscode-check` | Verify VS Code/Cursor setup | +| `claude-vscode-config` | Generate VS Code settings.json snippet | +| `claude-token-status` | Check token freshness and auto-refresh status | +| `claude-setup-token-refresh` | Enable hourly automatic token refresh | +| `claude-remove-token-refresh` | Disable automatic token refresh | + +## VS Code/Cursor Remote SSH Setup + +For remote development using VS Code or Cursor, use the built-in helpers: + +```bash +# Show complete setup guide +claude-vscode-setup + +# Get Python interpreter path +claude-vscode-env + +# Verify setup +claude-vscode-check + +# Generate VS Code settings.json +claude-vscode-config +``` + +### Quick Setup Steps + +1. **Install Remote SSH Extension** + - VS Code: Install "Remote - SSH" extension + - Cursor: Built-in (already included) + +2. **Configure Default Extensions** + - Command Palette → `Remote-SSH: Settings` + - Add: `ms-Python.python` and `ms-toolsai.jupyter` + +3. **Connect to Cluster** + - Command Palette → `Remote-SSH: Connect to Host` + +4. **Select Python Interpreter** + - Run `claude-vscode-env` to get the path + - Command Palette → `Python: Select Interpreter` + - Use the `pythonEnv-xxx` interpreter for full Databricks Runtime access + +**Important**: Regular Python `.py` files don't have access to Databricks globals (`dbutils`, `spark`). Only IPYNB notebooks and Databricks notebooks have this access. + +See the [VS Code Setup Script](scripts/vscode-setup.sh) for a standalone helper. + +## Troubleshooting + +### Offline Packages Not Found + +If the cluster fails to start with "Offline packages not found": + +1. Verify packages are uploaded: + ```bash + databricks fs ls dbfs:/init-scripts/offline-packages/ + ``` + +2. Check the path in cluster environment variables: + ```bash + # Should match OFFLINE_PACKAGES_PATH + echo $OFFLINE_PACKAGES_PATH + ``` + +3. Ensure the path is accessible from the cluster (DBFS mount point) + +### Init Script Fails + +Check cluster logs: +```bash +# Enable cluster log delivery in cluster config +# Then view: /cluster-logs//init_scripts/ +``` + +### Commands Not Found + +```bash +# Reload bashrc +source ~/.bashrc + +# Check PATH +echo $PATH | grep claude + +# Verify installation +check-claude +``` + +## Security Considerations + +### Volume Permissions + +Ensure appropriate Unity Catalog permissions: + +```sql +-- Grant read access to volume +GRANT READ VOLUME ON VOLUME .. TO ; +``` + +### Token Security + +- Tokens are **never hardcoded** in configs +- Read from environment: `$DATABRICKS_TOKEN` +- Configs regenerate per session +- Settings files are user-readable only (`~/.claude/`) + +## References + +- [Offline Installation Guide](scripts/OFFLINE-INSTALLATION.md) +- [Databricks Init Scripts Documentation](https://docs.databricks.com/init-scripts/) +- [Unity Catalog Volumes](https://docs.databricks.com/volumes/) +- [Databricks Terraform Provider](https://registry.terraform.io/providers/databricks/databricks/latest/docs) + +## License + +This module is provided as-is for use with Databricks workspaces. diff --git a/modules/adb-coding-assistants-cluster-offline/main.tf b/modules/adb-coding-assistants-cluster-offline/main.tf new file mode 100644 index 00000000..fcb66989 --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/main.tf @@ -0,0 +1,84 @@ +# Data source to get current user +data "databricks_current_user" "me" {} + +# Local value for init script path +locals { + init_script_path = var.init_script_source_path != null ? var.init_script_source_path : "${path.module}/scripts/install-claude-offline.sh" +} + +# Create or reference the volume for init scripts +resource "databricks_volume" "init_scripts" { + name = var.volume_name + catalog_name = var.catalog_name + schema_name = var.schema_name + volume_type = "MANAGED" + comment = "Volume for Claude Code CLI offline init scripts" + + lifecycle { + ignore_changes = [owner] + } +} + +# Upload the init script to the volume +resource "databricks_file" "init_script" { + source = local.init_script_path + path = "${databricks_volume.init_scripts.volume_path}/install-claude-offline.sh" +} + +# Create the cluster with init script +resource "databricks_cluster" "coding_assistants" { + cluster_name = var.cluster_name + spark_version = var.spark_version + node_type_id = var.node_type_id + autotermination_minutes = var.autotermination_minutes + data_security_mode = "SINGLE_USER" + single_user_name = data.databricks_current_user.me.user_name + + # Autoscaling or fixed size + dynamic "autoscale" { + for_each = var.num_workers == null ? [1] : [] + content { + min_workers = var.min_workers + max_workers = var.max_workers + } + } + + num_workers = var.num_workers + + # Single node configuration + spark_conf = var.cluster_mode == "SINGLE_NODE" ? { + "spark.databricks.cluster.profile" = "singleNode" + "spark.master" = "local[*]" + } : {} + + custom_tags = merge( + var.tags, + { + "ManagedBy" = "Terraform" + }, + var.cluster_mode == "SINGLE_NODE" ? { + "ResourceClass" = "SingleNode" + } : {} + ) + + # Environment variables for Claude Code CLI + spark_env_vars = merge( + { + MLFLOW_EXPERIMENT_NAME = var.mlflow_experiment_name + }, + var.offline_packages_path != null ? { + OFFLINE_PACKAGES_PATH = var.offline_packages_path + } : {} + ) + + # Init script configuration + init_scripts { + volumes { + destination = "${databricks_volume.init_scripts.volume_path}/install-claude-offline.sh" + } + } + + depends_on = [ + databricks_file.init_script + ] +} diff --git a/modules/adb-coding-assistants-cluster-offline/outputs.tf b/modules/adb-coding-assistants-cluster-offline/outputs.tf new file mode 100644 index 00000000..052478a1 --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/outputs.tf @@ -0,0 +1,39 @@ +output "cluster_id" { + description = "The ID of the created cluster" + value = databricks_cluster.coding_assistants.id +} + +output "cluster_url" { + description = "URL to access the cluster in Databricks UI" + value = databricks_cluster.coding_assistants.url +} + +output "cluster_name" { + description = "Name of the created cluster" + value = databricks_cluster.coding_assistants.cluster_name +} + +output "volume_path" { + description = "Path to the volume containing init scripts" + value = databricks_volume.init_scripts.volume_path +} + +output "volume_full_name" { + description = "Full name of the volume" + value = "${var.catalog_name}.${var.schema_name}.${var.volume_name}" +} + +output "init_script_path" { + description = "Path to the init script in the volume" + value = databricks_file.init_script.path +} + +output "mlflow_experiment_name" { + description = "MLflow experiment name for tracing" + value = var.mlflow_experiment_name +} + +output "offline_packages_path" { + description = "Path to offline packages directory" + value = var.offline_packages_path != null ? var.offline_packages_path : "/dbfs/init-scripts/offline-packages" +} diff --git a/modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md b/modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md new file mode 100644 index 00000000..af9d3674 --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/scripts/OFFLINE-INSTALLATION.md @@ -0,0 +1,220 @@ +# Offline Installation Guide for Claude Code CLI + +This guide explains how to install Claude Code CLI on Databricks clusters without internet access (air-gapped or firewalled environments). + +## Overview + +The offline installation process involves two phases: + +1. **Download Phase** (on machine with internet access) +2. **Installation Phase** (on air-gapped Databricks cluster) + +## Phase 1: Download Dependencies + +Run this on a machine with internet access: + +```bash +# Download all dependencies +bash download-offline-dependencies.sh + +# Optional: specify architecture (default: amd64) +bash download-offline-dependencies.sh arm64 + +# Create tarball for transfer +tar czf claude-offline-packages.tar.gz offline-packages/ +``` + +This will create an `offline-packages/` directory containing: + +``` +offline-packages/ +├── apt/ # System packages (curl, wget, git, jq) +├── python/ # Python wheels (MLflow and dependencies) +├── node/ # Node.js 20.x +├── claude/ # Claude Code CLI installer +└── manifest.txt # Package inventory +``` + +## Phase 2: Upload to Databricks + +### Option A: Using DBFS + +```bash +# Upload to DBFS +databricks fs cp -r offline-packages/ dbfs:/init-scripts/offline-packages/ + +# Or using tarball +databricks fs cp claude-offline-packages.tar.gz dbfs:/init-scripts/ +``` + +### Option B: Using Workspace Files + +```bash +# Upload via Databricks CLI +databricks workspace import-dir offline-packages/ /Workspace/Shared/init-scripts/offline-packages/ +``` + +### Option C: Using Azure Storage (for Azure Databricks) + +```bash +# Upload to storage account used by your workspace +az storage blob upload-batch \ + --account-name \ + --destination init-scripts \ + --source offline-packages/ +``` + +## Phase 3: Configure Cluster Init Script + +### Using DBFS Path + +```hcl +resource "databricks_cluster" "claude_cluster" { + # ... other configuration ... + + init_scripts { + dbfs { + destination = "dbfs:/init-scripts/install-claude-offline.sh" + } + } + + spark_env_vars = { + OFFLINE_PACKAGES_PATH = "/dbfs/init-scripts/offline-packages" + } +} +``` + +### Using Workspace Path + +```hcl +resource "databricks_cluster" "claude_cluster" { + # ... other configuration ... + + init_scripts { + workspace { + destination = "/Shared/init-scripts/install-claude-offline.sh" + } + } + + spark_env_vars = { + OFFLINE_PACKAGES_PATH = "/Workspace/Shared/init-scripts/offline-packages" + } +} +``` + +## Phase 4: Install on Cluster + +The offline installation script will automatically run during cluster startup. + +After the cluster starts, SSH into the cluster and verify: + +```bash +# Reload bashrc +source ~/.bashrc + +# Check installation +check-claude + +# Test Claude CLI +echo "what is 1+1?" | claude --print +``` + +## Troubleshooting + +### Missing Packages + +If packages are missing from the offline bundle: + +```bash +# On internet-connected machine, download specific package +cd offline-packages/python +pip download + +# Re-upload to DBFS +databricks fs cp .whl dbfs:/init-scripts/offline-packages/python/ +``` + +### Wrong Architecture + +If you get "wrong architecture" errors: + +```bash +# Download for correct architecture +bash download-offline-dependencies.sh arm64 # or amd64 + +# Re-upload packages +``` + +### Claude Installer Still Tries to Download + +The Claude installer may still attempt internet access. To fully offline install: + +1. On an internet-connected machine with same OS/architecture as cluster, install Claude +2. Copy the installed binary: + ```bash + # After installing Claude on test machine + cp ~/.local/bin/claude offline-packages/claude/claude-binary + cp -r ~/.claude offline-packages/claude/claude-config/ + ``` + +3. Modify `install-claude-offline.sh` to copy binary directly: + ```bash + # Add to install_claude_offline function: + if [ -f "$OFFLINE_PACKAGES_PATH/claude/claude-binary" ]; then + mkdir -p "$HOME/.local/bin" + cp "$OFFLINE_PACKAGES_PATH/claude/claude-binary" "$HOME/.local/bin/claude" + chmod +x "$HOME/.local/bin/claude" + log "✓ Claude Code installed from offline binary" + return 0 + fi + ``` + +### Node.js Installation Fails + +If Node.js from .deb fails due to dependencies: + +```bash +# Use the tarball method instead - it's dependency-free +# The script automatically tries this as fallback +``` + +## Size Estimates + +Approximate download sizes: + +- Node.js: ~30-40 MB +- APT packages: ~5-10 MB +- Python packages (MLflow): ~50-80 MB +- Claude installer: ~1 MB +- **Total: ~100-150 MB** + +## Security Considerations + +For high-security environments: + +1. **Verify checksums** of downloaded packages +2. **Scan packages** for vulnerabilities before upload +3. **Use internal artifact repository** (Artifactory, Nexus) +4. **Sign packages** if required by your organization +5. **Maintain version inventory** for compliance + +## Alternative: Internal Mirror + +For large-scale deployments, consider setting up internal mirrors: + +```bash +# Example: Internal PyPI mirror +pip install --index-url https://pypi.internal.company.com/simple mlflow[databricks] + +# Example: Internal NPM registry +npm config set registry https://npm.internal.company.com +``` + +Then modify the online installer to use internal mirrors instead of public repositories. + +## Support + +For issues specific to: +- **Claude CLI**: See [Claude AI Documentation](https://claude.ai/docs) +- **Databricks clusters**: Contact Databricks support +- **This script**: Open an issue in the repository diff --git a/modules/adb-coding-assistants-cluster-offline/scripts/download-offline-dependencies.sh b/modules/adb-coding-assistants-cluster-offline/scripts/download-offline-dependencies.sh new file mode 100755 index 00000000..5740328d --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/scripts/download-offline-dependencies.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Download offline dependencies for Claude Code CLI installation +# Run this on a machine with internet access, then copy the offline-packages directory to your air-gapped environment +# + +set -euo pipefail + +OFFLINE_DIR="offline-packages" +ARCH="${1:-amd64}" # amd64 or arm64 + +echo "=== Downloading Offline Dependencies for Claude Code CLI ===" +echo "Architecture: $ARCH" +echo "Output directory: $OFFLINE_DIR" +echo "" + +# Create directory structure +mkdir -p "$OFFLINE_DIR"/{apt,python,node,claude} + +# Download Node.js packages +echo "[1/4] Downloading Node.js 20.x..." +cd "$OFFLINE_DIR/node" +if [ "$ARCH" = "amd64" ]; then + NODE_VERSION="20.11.1" + wget -q "https://deb.nodesource.com/node_20.x/pool/main/n/nodejs/nodejs_${NODE_VERSION}-1nodesource1_amd64.deb" || { + echo "⚠ Failed to download Node.js. Trying alternative method..." + # Alternative: download from official Node.js + wget -q "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" -O node-linux-x64.tar.xz + } +elif [ "$ARCH" = "arm64" ]; then + NODE_VERSION="20.11.1" + wget -q "https://deb.nodesource.com/node_20.x/pool/main/n/nodejs/nodejs_${NODE_VERSION}-1nodesource1_arm64.deb" || { + echo "⚠ Failed to download Node.js. Trying alternative method..." + wget -q "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-arm64.tar.xz" -O node-linux-arm64.tar.xz + } +fi +cd - > /dev/null +echo "✓ Node.js downloaded" + +# Download APT packages +echo "[2/4] Downloading APT packages (curl, wget, git, jq)..." +cd "$OFFLINE_DIR/apt" +apt-get download curl wget git jq 2>/dev/null || { + echo "⚠ apt-get download failed. Downloading manually..." + # Fallback: download from packages.ubuntu.com + UBUNTU_VERSION="jammy" # Ubuntu 22.04 + wget -q "http://archive.ubuntu.com/ubuntu/pool/main/c/curl/curl_7.81.0-1ubuntu1.15_${ARCH}.deb" 2>/dev/null || true + wget -q "http://archive.ubuntu.com/ubuntu/pool/main/w/wget/wget_1.21.2-2ubuntu1_${ARCH}.deb" 2>/dev/null || true + wget -q "http://archive.ubuntu.com/ubuntu/pool/main/g/git/git_2.34.1-1ubuntu1.10_${ARCH}.deb" 2>/dev/null || true + wget -q "http://archive.ubuntu.com/ubuntu/pool/universe/j/jq/jq_1.6-2.1ubuntu3_${ARCH}.deb" 2>/dev/null || true +} +cd - > /dev/null +echo "✓ APT packages downloaded" + +# Download Python packages +echo "[3/4] Downloading Python packages (MLflow)..." +cd "$OFFLINE_DIR/python" +pip download "mlflow[databricks]>=3.4" --dest . 2>/dev/null || { + echo "⚠ pip download failed. Make sure pip is installed." +} +cd - > /dev/null +echo "✓ Python packages downloaded" + +# Download Claude CLI +echo "[4/4] Downloading Claude Code CLI installer..." +cd "$OFFLINE_DIR/claude" +wget -q https://claude.ai/install.sh -O install.sh || { + echo "⚠ Failed to download Claude installer" +} +chmod +x install.sh +cd - > /dev/null +echo "✓ Claude installer downloaded" + +# Create manifest +cat > "$OFFLINE_DIR/manifest.txt" </dev/null 2>&1; } + +# Path to offline packages (can be overridden via environment variable) +OFFLINE_PACKAGES_PATH="${OFFLINE_PACKAGES_PATH:-/dbfs/init-scripts/offline-packages}" + +# Validate offline packages exist +if [ ! -d "$OFFLINE_PACKAGES_PATH" ]; then + log "✗ ERROR: Offline packages not found at: $OFFLINE_PACKAGES_PATH" + log " Please upload offline-packages directory to DBFS and set OFFLINE_PACKAGES_PATH" + exit 1 +fi + +log "Using offline packages from: $OFFLINE_PACKAGES_PATH" + +# Install Node.js from local package +install_nodejs_offline() { + if cmd_exists node && cmd_exists npm; then + log "✓ Node.js already installed ($(node --version))" + return 0 + fi + + log "Installing Node.js from offline packages..." + + # Try .deb package first + if ls "$OFFLINE_PACKAGES_PATH"/node/*.deb 1> /dev/null 2>&1; then + if sudo dpkg -i "$OFFLINE_PACKAGES_PATH"/node/*.deb &>>$L; then + log "✓ Node.js installed from .deb package" + return 0 + fi + fi + + # Try .tar.xz package + if ls "$OFFLINE_PACKAGES_PATH"/node/*.tar.xz 1> /dev/null 2>&1; then + local NODE_TAR=$(ls "$OFFLINE_PACKAGES_PATH"/node/*.tar.xz | head -1) + sudo tar -xJf "$NODE_TAR" -C /usr/local --strip-components=1 &>>$L + if cmd_exists node && cmd_exists npm; then + log "✓ Node.js installed from tarball" + return 0 + fi + fi + + log "⚠ Node.js installation failed" + return 1 +} + +# Install Claude Code CLI from local installer +install_claude_offline() { + if cmd_exists claude; then + log "✓ Claude Code already installed" + return 0 + fi + + log "Installing Claude Code CLI from offline package..." + + if [ -f "$OFFLINE_PACKAGES_PATH/claude/install.sh" ]; then + # Run the installer - it may still try to download, but at least we have it locally + if bash "$OFFLINE_PACKAGES_PATH/claude/install.sh" &>>$L; then + log "✓ Claude Code installation completed" + return 0 + fi + fi + + log "⚠ Claude Code installation failed" + return 1 +} + +# Install APT packages from local cache +install_apt_packages_offline() { + log "Installing system dependencies from offline packages..." + + if [ -d "$OFFLINE_PACKAGES_PATH/apt" ] && [ "$(ls -A $OFFLINE_PACKAGES_PATH/apt)" ]; then + # Install dependencies first to avoid dpkg errors + if sudo dpkg -i "$OFFLINE_PACKAGES_PATH"/apt/*.deb &>>$L; then + log "✓ System dependencies installed from offline packages" + return 0 + else + # Try to fix broken dependencies + sudo apt-get install -f -y &>>$L || true + log "⚠ Some packages may have failed to install" + fi + else + log "⚠ No APT packages found in offline directory" + fi + + return 0 +} + +# Install Python packages from local wheels +install_python_packages_offline() { + log "Installing MLflow from offline packages..." + + if [ -d "$OFFLINE_PACKAGES_PATH/python" ] && [ "$(ls -A $OFFLINE_PACKAGES_PATH/python)" ]; then + if pip install --no-index --find-links="$OFFLINE_PACKAGES_PATH/python" "mlflow[databricks]" &>>$L; then + log "✓ MLflow installed from offline packages" + return 0 + else + log "⚠ MLflow installation failed" + fi + else + log "⚠ No Python packages found in offline directory" + fi + + return 1 +} + +# Add helper functions to bashrc (same as online version) +setup_bashrc() { + local START_MARKER="### CLAUDE_CODE_HELPERS_START ###" + local END_MARKER="### CLAUDE_CODE_HELPERS_END ###" + + # Backup bashrc + [ -f "$HOME/.bashrc" ] && cp "$HOME/.bashrc" "$HOME/.bashrc.backup-$(date +%s)" + + # Remove any existing Claude sections (between markers) + if [ -f "$HOME/.bashrc" ]; then + if grep -q "$START_MARKER" "$HOME/.bashrc" 2>/dev/null; then + log "Removing old bashrc helpers..." + sed -i "/$START_MARKER/,/$END_MARKER/d" "$HOME/.bashrc" + fi + fi + + W="${DATABRICKS_HOST}" + E="${MLFLOW_EXPERIMENT_NAME:-/Workspace/Shared/claude-code-tracing}" + + log "Adding helpers to bashrc..." + + cat >> "$HOME/.bashrc" <<'EOF' + +### CLAUDE_CODE_HELPERS_START ### +# Claude Code CLI Setup (auto-generated - do not edit manually) +export PATH="$HOME/.claude/bin:$HOME/.local/bin:$PATH" + +# Claude Code MLflow tracing helpers +export DATABRICKS_HOST="${DATABRICKS_HOST:-WS_PH}" +export MLFLOW_EXPERIMENT_NAME="${MLFLOW_EXPERIMENT_NAME:-EXP_PH}" + +# Set Anthropic environment variables for Claude CLI +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + export ANTHROPIC_AUTH_TOKEN="$DATABRICKS_TOKEN" + export ANTHROPIC_BASE_URL="${DATABRICKS_HOST}/serving-endpoints/anthropic" + export ANTHROPIC_MODEL="databricks-claude-sonnet-4-5" + export ANTHROPIC_CUSTOM_HEADERS="x-databricks-disable-beta-headers: true" +fi + +# Internal function to generate Claude settings +_generate_claude_config() { + local config_file="$HOME/.claude/settings.json" + + cat > "$config_file" </dev/null 2>&1; then + if ! jq empty "$config_file" 2>/dev/null; then + echo "⚠ Claude settings JSON validation failed" >&2 + return 1 + fi + fi + + # Store token hash for change detection + if [ -n "$DATABRICKS_TOKEN" ]; then + echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 > "$HOME/.claude/.token_hash" 2>/dev/null || true + fi + + return 0 +} + +# Check if token has changed and refresh if needed +_check_and_refresh_token() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + return 0 # Skip if token not available + fi + + local config_file="$HOME/.claude/settings.json" + local token_hash_file="$HOME/.claude/.token_hash" + + # Calculate current token hash + local current_hash + current_hash=$(echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 2>/dev/null || echo "") + + if [ -z "$current_hash" ]; then + return 0 # Skip if hash calculation failed + fi + + # Check if token has changed + if [ -f "$token_hash_file" ]; then + local stored_hash + stored_hash=$(cat "$token_hash_file" 2>/dev/null || echo "") + if [ "$current_hash" = "$stored_hash" ]; then + return 0 # Token unchanged, no refresh needed + fi + fi + + # Token changed or first time - refresh config + mkdir -p "$HOME/.claude" + if _generate_claude_config >/dev/null 2>&1; then + # Only show message if in interactive shell (not cron) + if [ -t 0 ]; then + echo "✓ Claude Code token refreshed automatically" + fi + return 0 + fi + + return 1 +} + +# Auto-generate Claude settings from environment on first login +if [ ! -f "$HOME/.claude/settings.json" ] && [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + mkdir -p "$HOME/.claude" + if _generate_claude_config; then + echo "✓ Claude Code settings.json created" + fi +fi + +# Auto-refresh token on shell login if it has changed +# This ensures settings.json stays in sync with DATABRICKS_TOKEN +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + _check_and_refresh_token +fi + +# Regenerate Claude settings from current environment +claude-refresh-token() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + echo "⚠ DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + return 1 + fi + + mkdir -p "$HOME/.claude" + if _generate_claude_config; then + echo "✓ Claude Code settings updated" + else + echo "⚠ Failed to update Claude settings" + return 1 + fi +} + +# Setup cron job for periodic token refresh (runs hourly) +claude-setup-token-refresh() { + local cron_file="$HOME/.claude/token-refresh-cron" + + # Create cron wrapper script + mkdir -p "$HOME/.claude" + cat > "$cron_file" <<'CRON_SCRIPT' +#!/bin/bash +# Auto-generated cron script for Claude token refresh +# This script is called by cron to refresh the Claude token periodically + +# Source bashrc to get functions +if [ -f "$HOME/.bashrc" ]; then + source "$HOME/.bashrc" >/dev/null 2>&1 +fi + +# Check and refresh token if needed +_check_and_refresh_token +CRON_SCRIPT + chmod +x "$cron_file" + + # Check if cron job already exists + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + echo "✓ Token refresh cron job already configured" + return 0 + fi + + # Add cron job + (crontab -l 2>/dev/null; echo "0 * * * * $cron_file") | crontab - + if [ $? -eq 0 ]; then + echo "✓ Token refresh cron job configured (runs hourly)" + echo " To remove: crontab -e" + else + echo "⚠ Failed to setup cron job (may require cron service)" + return 1 + fi +} + +# Remove token refresh cron job +claude-remove-token-refresh() { + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + crontab -l 2>/dev/null | grep -v "token-refresh-cron" | crontab - + echo "✓ Token refresh cron job removed" + else + echo "ℹ No token refresh cron job found" + fi +} + +# Check token freshness status +claude-token-status() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + echo "⚠ DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + return 1 + fi + + local config_file="$HOME/.claude/settings.json" + local token_hash_file="$HOME/.claude/.token_hash" + + echo "=== Claude Token Status ===" + echo "" + + # Check if config file exists + if [ -f "$config_file" ]; then + echo "✓ Settings file: $config_file" + local file_age + file_age=$(stat -c %Y "$config_file" 2>/dev/null || stat -f %m "$config_file" 2>/dev/null || echo "0") + local current_time + current_time=$(date +%s) + local age_hours + age_hours=$(( (current_time - file_age) / 3600 )) + echo " Last updated: ${age_hours} hour(s) ago" + else + echo "✗ Settings file: missing" + fi + + echo "" + + # Check token hash + if [ -f "$token_hash_file" ]; then + local current_hash + current_hash=$(echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 2>/dev/null || echo "") + local stored_hash + stored_hash=$(cat "$token_hash_file" 2>/dev/null || echo "") + if [ "$current_hash" = "$stored_hash" ] && [ -n "$current_hash" ]; then + echo "✓ Token: matches stored hash (up to date)" + else + echo "⚠ Token: differs from stored hash (needs refresh)" + echo " Run: claude-refresh-token" + fi + else + echo "ℹ Token hash: not stored (will be created on next refresh)" + fi + + echo "" + + # Check cron job + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + echo "✓ Auto-refresh: enabled (hourly cron job)" + else + echo "ℹ Auto-refresh: disabled" + echo " Enable with: claude-setup-token-refresh" + fi +} + +claude-tracing-enable() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + echo "⚠ DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + return 1 + fi + + if ! command -v mlflow >/dev/null 2>&1; then + echo "⚠ MLflow is not installed" + return 1 + fi + + python3 </dev/null 2>&1; then + echo "✓ Claude Code CLI: $(which claude)" + else + echo "✗ Claude Code CLI: not found" + fi + echo "" + [ -f "$HOME/.claude/settings.json" ] && echo "✓ Settings configured" || echo "✗ Settings missing" + [ -n "$ANTHROPIC_AUTH_TOKEN" ] && echo "✓ Authentication configured" || echo "✗ Authentication not set" + echo "" + + # VS Code/Cursor Remote SSH info + echo "VS Code/Cursor Remote SSH:" + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo " ✓ Python virtual environment: $venv_path" + echo " Run 'claude-vscode-setup' for setup instructions" + else + echo " ℹ Run 'claude-vscode-setup' for Remote SSH setup guide" + fi +} + +claude-debug() { + echo "=== Claude CLI Debug Info ===" + [ -f "$HOME/.claude/settings.json" ] && cat "$HOME/.claude/settings.json" || echo "Settings missing!" + echo "" + env | grep -E "ANTHROPIC|DATABRICKS" || echo "No env vars" +} + +# VS Code/Cursor Remote SSH helpers +claude-vscode-env() { + # Show the Databricks virtual environment path for VS Code/Cursor + if [ -n "$DATABRICKS_VIRTUAL_ENV" ]; then + echo "$DATABRICKS_VIRTUAL_ENV" + else + # Try to find pythonEnv-* directories + local python_envs + python_envs=$(find /databricks/python* -maxdepth 1 -type d -name "pythonEnv-*" 2>/dev/null | head -1) + if [ -n "$python_envs" ]; then + echo "$python_envs" + else + echo "⚠ DATABRICKS_VIRTUAL_ENV not set and pythonEnv-* not found" + echo " Try: echo \$DATABRICKS_VIRTUAL_ENV" + return 1 + fi + fi +} + +claude-vscode-setup() { + echo "=== VS Code/Cursor Remote SSH Setup Guide ===" + echo "" + echo "1. Install Remote SSH Extension" + echo " - VS Code: Install 'Remote - SSH' extension" + echo " - Cursor: Built-in Remote SSH extension (already included)" + echo "" + echo "2. Configure Default Extensions" + echo " Open Command Palette (Cmd+Shift+P / Ctrl+Shift+P):" + echo " → Remote-SSH: Settings" + echo "" + echo " Or edit settings.json and add:" + echo "" + cat <<'VSCODE_SETTINGS' + "remote.SSH.defaultExtensions": [ + "ms-Python.python", + "ms-toolsai.jupyter" + ] +VSCODE_SETTINGS + echo "" + echo "3. Connect to Cluster" + echo " - Command Palette → Remote-SSH: Connect to Host" + echo " - Enter your cluster SSH connection details" + echo "" + echo "4. Select Python Interpreter" + echo " After connecting, run this command to get the Python path:" + echo "" + echo " $ claude-vscode-env" + echo "" + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo " Current virtual environment:" + echo " $venv_path" + echo "" + echo " Then in VS Code/Cursor:" + echo " - Command Palette → Python: Select Interpreter" + echo " - Paste the path above or browse to it" + else + echo " Run 'echo \$DATABRICKS_VIRTUAL_ENV' to find the path" + fi + echo "" + echo "5. Important Notes" + echo " • IPYNB notebooks and *.py Databricks notebooks have access to" + echo " Databricks globals (dbutils, spark, etc.)" + echo " • Regular Python *.py files do NOT have access to Databricks globals" + echo " • Always select the pythonEnv-xxx interpreter for full Databricks" + echo " Runtime library access" + echo "" + echo "6. Verify Setup" + echo " Run: claude-vscode-check" +} + +claude-vscode-check() { + echo "=== VS Code/Cursor Remote SSH Setup Check ===" + echo "" + + # Check for virtual environment + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "✓ Python Virtual Environment:" + echo " $venv_path" + if [ -d "$venv_path/bin" ]; then + echo " ✓ Virtual environment directory exists" + if [ -f "$venv_path/bin/python" ]; then + echo " ✓ Python executable found" + echo " Python version: $($venv_path/bin/python --version 2>&1 || echo 'unknown')" + else + echo " ⚠ Python executable not found" + fi + else + echo " ⚠ Virtual environment directory not found" + fi + else + echo "✗ Python Virtual Environment: Not found" + echo " Run: echo \$DATABRICKS_VIRTUAL_ENV" + fi + echo "" + + # Check for Python + if command -v python3 >/dev/null 2>&1; then + echo "✓ Python3 available: $(which python3)" + echo " Version: $(python3 --version 2>&1)" + else + echo "✗ Python3 not found in PATH" + fi + echo "" + + # Check for Databricks runtime libraries + echo "Databricks Runtime Libraries:" + python3 <<'PYTHON_CHECK' +import sys +libraries = ['pyspark', 'pandas', 'numpy', 'mlflow', 'databricks'] +found = [] +missing = [] + +for lib in libraries: + try: + __import__(lib) + found.append(lib) + except ImportError: + missing.append(lib) + +if found: + print(f" ✓ Available: {', '.join(found)}") +if missing: + print(f" ⚠ Missing: {', '.join(missing)}") + +# Check for Databricks globals (only available in notebooks) +try: + import dbutils + print(" ✓ dbutils available (notebook context)") +except: + print(" ℹ dbutils not available (normal for .py files)") +PYTHON_CHECK + + echo "" + echo "VS Code/Cursor Configuration:" + echo " Run 'claude-vscode-setup' for setup instructions" + echo " Run 'claude-vscode-env' to get Python interpreter path" +} + +claude-vscode-config() { + # Generate VS Code settings.json snippet + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + + echo "=== VS Code/Cursor settings.json Configuration ===" + echo "" + echo "Add this to your VS Code/Cursor settings.json:" + echo "" + echo "{" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "," + echo " \"python.defaultInterpreterPath\": \"$venv_path/bin/python\"" + fi + echo "}" + echo "" + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "Python interpreter path:" + echo " $venv_path/bin/python" + echo "" + echo "To set this in VS Code/Cursor:" + echo " 1. Command Palette → Python: Select Interpreter" + echo " 2. Enter interpreter path: $venv_path/bin/python" + else + echo "To find Python interpreter path, run:" + echo " claude-vscode-env" + fi +} +### CLAUDE_CODE_HELPERS_END ### +EOF + + sed -i "s|WS_PH|$W|g; s|EXP_PH|$E|g" "$HOME/.bashrc" + log "✓ Bashrc helpers added" +} + +# Main installation +main() { + log "Starting offline installation..." + log "Offline packages: $OFFLINE_PACKAGES_PATH" + + # Install from offline packages + install_apt_packages_offline + install_python_packages_offline + install_nodejs_offline + install_claude_offline + + # Configure + if setup_bashrc; then + log "✓ Bashrc configuration completed" + fi + + log "" + log "=== Installation Summary ===" + log "Installation complete. Full log: $L" + log "" + log "Next steps (on cluster login):" + log " 1. Run: source ~/.bashrc" + log " 2. Verify: check-claude" + log " 3. Use: claude command" + # Setup automatic token refresh (optional - user can enable manually) + log "Setting up automatic token refresh..." + if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + # Setup cron job for periodic refresh + if command -v crontab >/dev/null 2>&1; then + # Source bashrc temporarily to get the function + source "$HOME/.bashrc" >/dev/null 2>&1 || true + claude-setup-token-refresh >/dev/null 2>&1 || log "⚠ Cron setup skipped (may require manual setup)" + else + log "⚠ Cron not available - token refresh will only happen on login" + fi + fi + + log "" + log "Helper commands:" + log " - check-claude: Verify installation" + log " - claude-debug: Show configuration" + log " - claude-refresh-token: Update settings" + log " - claude-token-status: Check token freshness and auto-refresh status" + log " - claude-setup-token-refresh: Enable hourly automatic token refresh" + log " - claude-remove-token-refresh: Disable automatic token refresh" + log " - claude-tracing-enable/disable/status: Manage tracing" + log " - claude-vscode-setup: Show VS Code/Cursor Remote SSH setup guide" + log " - claude-vscode-env: Get Python virtual environment path" + log " - claude-vscode-check: Verify VS Code/Cursor setup" + log " - claude-vscode-config: Generate VS Code settings.json snippet" + return 0 +} + +main +exit 0 diff --git a/modules/adb-coding-assistants-cluster-offline/scripts/vscode-setup.sh b/modules/adb-coding-assistants-cluster-offline/scripts/vscode-setup.sh new file mode 100755 index 00000000..89b21ada --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/scripts/vscode-setup.sh @@ -0,0 +1,248 @@ +#!/bin/bash +# +# VS Code/Cursor Remote SSH Setup Helper for Databricks Clusters +# This script helps configure VS Code or Cursor for remote development on Databricks clusters +# + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +log_info() { echo -e "${BLUE}ℹ${NC} $1"; } +log_success() { echo -e "${GREEN}✓${NC} $1"; } +log_warning() { echo -e "${YELLOW}⚠${NC} $1"; } +log_error() { echo -e "${RED}✗${NC} $1"; } + +# Find Databricks Python virtual environment +find_python_env() { + if [ -n "${DATABRICKS_VIRTUAL_ENV:-}" ]; then + echo "$DATABRICKS_VIRTUAL_ENV" + return 0 + fi + + # Try to find pythonEnv-* directories + local python_envs + python_envs=$(find /databricks/python* -maxdepth 1 -type d -name "pythonEnv-*" 2>/dev/null | head -1) + if [ -n "$python_envs" ]; then + echo "$python_envs" + return 0 + fi + + return 1 +} + +# Show setup instructions +show_setup_guide() { + echo "==========================================" + echo "VS Code/Cursor Remote SSH Setup Guide" + echo "==========================================" + echo "" + + echo "1. Install Remote SSH Extension" + echo " • VS Code: Install 'Remote - SSH' extension from marketplace" + echo " • Cursor: Built-in Remote SSH extension (already included)" + echo "" + + echo "2. Configure Default Extensions" + echo " Open Command Palette (Cmd+Shift+P / Ctrl+Shift+P):" + echo " → Type: Remote-SSH: Settings" + echo "" + echo " Or manually edit settings.json:" + echo "" + echo " {" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + echo " }" + echo "" + + echo "3. Connect to Cluster" + echo " • Command Palette → Remote-SSH: Connect to Host" + echo " • Enter your cluster SSH connection details" + echo " • Format: user@hostname or use SSH config entry" + echo "" + + echo "4. Select Python Interpreter" + local venv_path + if venv_path=$(find_python_env 2>/dev/null); then + echo " ✓ Found Python virtual environment:" + echo " $venv_path" + echo "" + echo " In VS Code/Cursor:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Enter interpreter path:" + echo " $venv_path/bin/python" + echo "" + echo " Or copy this path:" + echo " $venv_path/bin/python" + else + echo " ⚠ Could not auto-detect Python virtual environment" + echo " Run this command to find it:" + echo " echo \$DATABRICKS_VIRTUAL_ENV" + echo "" + echo " Then in VS Code/Cursor:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Paste the path from above" + fi + echo "" + + echo "5. Important Notes" + echo " • IPYNB notebooks and *.py Databricks notebooks have access to" + echo " Databricks globals (dbutils, spark, etc.)" + echo " • Regular Python *.py files do NOT have access to Databricks globals" + echo " • Always select the pythonEnv-xxx interpreter for full Databricks" + echo " Runtime library access (pyspark, pandas, numpy, mlflow, etc.)" + echo "" + + echo "6. Verify Setup" + echo " After connecting, verify Python interpreter:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Should show: pythonEnv-xxx/bin/python" + echo "" + echo " Test in a Python file:" + echo " import pyspark" + echo " import pandas" + echo " print('Setup successful!')" +} + +# Generate VS Code settings.json snippet +generate_settings() { + local venv_path + venv_path=$(find_python_env 2>/dev/null || echo "") + + echo "{" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + if [ -n "$venv_path" ]; then + echo "," + echo " \"python.defaultInterpreterPath\": \"$venv_path/bin/python\"" + fi + echo "}" +} + +# Check current setup +check_setup() { + echo "==========================================" + echo "VS Code/Cursor Setup Check" + echo "==========================================" + echo "" + + # Check for virtual environment + local venv_path + if venv_path=$(find_python_env 2>/dev/null); then + log_success "Python Virtual Environment found:" + echo " $venv_path" + + if [ -d "$venv_path/bin" ]; then + log_success "Virtual environment directory exists" + if [ -f "$venv_path/bin/python" ]; then + log_success "Python executable found" + echo " Python version: $($venv_path/bin/python --version 2>&1 || echo 'unknown')" + else + log_warning "Python executable not found" + fi + else + log_warning "Virtual environment directory not found" + fi + else + log_error "Python Virtual Environment not found" + echo " Run: echo \$DATABRICKS_VIRTUAL_ENV" + fi + echo "" + + # Check for Python + if command -v python3 >/dev/null 2>&1; then + log_success "Python3 available: $(which python3)" + echo " Version: $(python3 --version 2>&1)" + else + log_error "Python3 not found in PATH" + fi + echo "" + + # Check for Databricks runtime libraries + echo "Databricks Runtime Libraries:" + python3 <<'PYTHON_CHECK' +import sys +libraries = ['pyspark', 'pandas', 'numpy', 'mlflow'] +found = [] +missing = [] + +for lib in libraries: + try: + __import__(lib) + found.append(lib) + except ImportError: + missing.append(lib) + +if found: + print(f" ✓ Available: {', '.join(found)}") +if missing: + print(f" ⚠ Missing: {', '.join(missing)}") + +# Check for Databricks globals (only available in notebooks) +try: + import dbutils + print(" ✓ dbutils available (notebook context)") +except: + print(" ℹ dbutils not available (normal for .py files)") +PYTHON_CHECK + + echo "" + echo "Next steps:" + echo " • Run this script with --guide to see setup instructions" + echo " • Run this script with --settings to generate settings.json" +} + +# Main +main() { + case "${1:-}" in + --guide|-g) + show_setup_guide + ;; + --settings|-s) + generate_settings + ;; + --check|-c) + check_setup + ;; + --env|-e) + find_python_env || { + log_error "Could not find Python virtual environment" + echo "Try: echo \$DATABRICKS_VIRTUAL_ENV" + exit 1 + } + ;; + --help|-h|"") + echo "VS Code/Cursor Remote SSH Setup Helper" + echo "" + echo "Usage: $0 [OPTION]" + echo "" + echo "Options:" + echo " --guide, -g Show complete setup guide" + echo " --settings, -s Generate VS Code settings.json snippet" + echo " --check, -c Check current setup status" + echo " --env, -e Show Python virtual environment path" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 --guide # Show setup instructions" + echo " $0 --env # Get Python interpreter path" + echo " $0 --check # Verify setup" + ;; + *) + log_error "Unknown option: $1" + echo "Run '$0 --help' for usage information" + exit 1 + ;; + esac +} + +main "$@" diff --git a/modules/adb-coding-assistants-cluster-offline/variables.tf b/modules/adb-coding-assistants-cluster-offline/variables.tf new file mode 100644 index 00000000..4388d30a --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/variables.tf @@ -0,0 +1,95 @@ +variable "cluster_name" { + description = "Name of the Databricks cluster" + type = string +} + +variable "catalog_name" { + description = "Unity Catalog name for the volume" + type = string +} + +variable "schema_name" { + description = "Schema name for the volume" + type = string + default = "default" +} + +variable "volume_name" { + description = "Volume name to store init scripts" + type = string + default = "coding_assistants" +} + +variable "init_script_source_path" { + description = "Local path to the init script" + type = string + default = null +} + +variable "offline_packages_path" { + description = "Path to offline packages directory (e.g., /dbfs/init-scripts/offline-packages). If not set, defaults to /dbfs/init-scripts/offline-packages" + type = string + default = null +} + +variable "spark_version" { + description = "Databricks Runtime version" + type = string + default = "14.3.x-scala2.12" +} + +variable "node_type_id" { + description = "Node type for the cluster" + type = string + default = "Standard_DS3_v2" +} + +variable "autotermination_minutes" { + description = "Minutes of inactivity before cluster auto-terminates" + type = number + default = 30 +} + +variable "num_workers" { + description = "Number of worker nodes (null for autoscaling)" + type = number + default = null +} + +variable "min_workers" { + description = "Minimum number of workers for autoscaling" + type = number + default = 1 +} + +variable "max_workers" { + description = "Maximum number of workers for autoscaling" + type = number + default = 3 +} + +variable "mlflow_experiment_name" { + description = "MLflow experiment name for Claude Code tracing" + type = string + default = "/Workspace/Shared/claude-code-tracing" +} + +variable "cluster_mode" { + description = "Cluster mode: STANDARD or SINGLE_NODE" + type = string + default = "STANDARD" + + validation { + condition = contains(["STANDARD", "SINGLE_NODE"], var.cluster_mode) + error_message = "cluster_mode must be either STANDARD or SINGLE_NODE" + } +} + +variable "tags" { + description = "Custom tags for the cluster" + type = map(string) + default = { + Environment = "dev" + Purpose = "coding-assistants-offline" + } +} diff --git a/modules/adb-coding-assistants-cluster-offline/versions.tf b/modules/adb-coding-assistants-cluster-offline/versions.tf new file mode 100644 index 00000000..07223296 --- /dev/null +++ b/modules/adb-coding-assistants-cluster-offline/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + databricks = { + source = "databricks/databricks" + version = ">= 1.40.0" + } + } +} diff --git a/modules/adb-coding-assistants-cluster/Makefile b/modules/adb-coding-assistants-cluster/Makefile new file mode 100644 index 00000000..653039d8 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/Makefile @@ -0,0 +1,7 @@ +.PHONY: docs test_docs + +docs: + terraform-docs -c ../../.terraform-docs.yml . + +test_docs: + terraform-docs -c ../../.terraform-docs.yml --output-check . diff --git a/modules/adb-coding-assistants-cluster/README.md b/modules/adb-coding-assistants-cluster/README.md new file mode 100644 index 00000000..8838f877 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/README.md @@ -0,0 +1,461 @@ +# Provisioning Databricks Cluster with Claude Code CLI + +This module deploys a Databricks cluster pre-configured with Claude Code CLI for AI-assisted development directly on Databricks. + +## Module content + +This module can be used to deploy the following: + +* Unity Catalog Volume for secure init script storage +* Init script with Claude Code CLI installation +* Databricks cluster with automatic AI coding assistant setup +* MLflow experiment configuration for tracing +* Helper bash functions for cluster users + +## Features + +- ✅ **Zero-configuration AI coding tools** on cluster startup +- ✅ **Unity Catalog Volumes** for secure script storage (Databricks recommended practice) +- ✅ **MLflow tracing** integration for Claude Code sessions +- ✅ **Flexible cluster configuration** (single-node or autoscaling) + +> **Note**: For offline/air-gapped environments, use the separate [`adb-coding-assistants-cluster-offline`](../adb-coding-assistants-cluster-offline/README.md) module. + +## Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ Unity Catalog Volume │ +│ /Volumes//// │ +│ └── install-claude.sh │ +└─────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────┐ +│ Databricks Cluster (on startup) │ +│ │ +│ 1. Executes init script from volume │ +│ 2. Installs Node.js, OpenCode, Claude CLI │ +│ 3. Configures bashrc with helper functions │ +│ 4. Auto-generates configs on user login │ +└─────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────┐ +│ User Login │ +│ │ +│ • DATABRICKS_TOKEN available from environment │ +│ • Configs auto-generate: │ +│ - ~/.claude/settings.json │ +│ - ~/.opencode/config.json │ +│ • Commands ready: claude, opencode │ +└─────────────────────────────────────────────────────┘ +``` + +## Prerequisites + +- Databricks workspace with Unity Catalog enabled +- Databricks Runtime 13.3 LTS or higher (recommended for Unity Catalog volumes) +- Databricks Terraform provider >= 1.40.0 +- Unity Catalog with an existing catalog and schema +- **Unity Catalog metastore must have a root storage credential configured** (required for volumes) + +> **Note**: If you encounter an error about missing root storage credential, you need to configure the metastore's root storage credential first. See [Databricks documentation](https://docs.databricks.com/api-explorer/workspace/metastores/update) for details. + +## Usage + +### Basic Example + +```hcl +module "coding_cluster" { + source = "./modules/coding-assistants-cluster" + + cluster_name = "ai-dev-cluster" + catalog_name = "main" + schema_name = "default" + + # init_script_source_path is optional - module includes the script +} +``` + +### Single-Node Cluster + +```hcl +module "single_node_cluster" { + source = "./modules/coding-assistants-cluster" + + cluster_name = "ai-dev-single-node" + catalog_name = "main" + schema_name = "default" + + cluster_mode = "SINGLE_NODE" + num_workers = 0 +} +``` + +### Autoscaling Cluster + +```hcl +module "autoscaling_cluster" { + source = "./modules/coding-assistants-cluster" + + cluster_name = "ai-dev-autoscaling" + catalog_name = "main" + schema_name = "default" + + min_workers = 2 + max_workers = 8 + + tags = { + Environment = "production" + Team = "data-science" + } +} +``` + +### Complete Example + +```hcl +module "coding_cluster" { + source = "./modules/coding-assistants-cluster" + + # Cluster configuration + cluster_name = "ai-development-cluster" + spark_version = "17.3.x-cpu-ml-scala2.13" + node_type_id = "Standard_D8pds_v6" + autotermination_minutes = 60 + + # Volume configuration + catalog_name = "main" + schema_name = "default" + volume_name = "coding_assistants" + + # Init script (optional - uses bundled script by default) + # init_script_source_path = "/path/to/custom/script.sh" + + # MLflow configuration + mlflow_experiment_name = "/Users/me@company.com/my-claude-traces" + + # Autoscaling + min_workers = 1 + max_workers = 5 + + # Tags + tags = { + Environment = "development" + Project = "ai-assisted-coding" + CostCenter = "engineering" + ManagedBy = "terraform" + } +} +``` + +## Init Script Storage Best Practices + +According to [Databricks documentation](https://docs.databricks.com/aws/en/init-scripts/): + +> **Databricks Runtime 13.3 LTS and above with Unity Catalog** +> Store init scripts in Unity Catalog volumes. + +### Why Unity Catalog Volumes? + +1. **Governance**: Full Unity Catalog ACL support +2. **Security**: Identity-based access control +3. **Portability**: Works across AWS, Azure, and GCP +4. **Versioning**: Easy to manage and update scripts +5. **No DBFS**: Recommended alternative to legacy DBFS storage + +### Init Script Identity + +- **Single-user access mode**: Uses assigned principal's identity +- **Standard access mode**: Uses cluster owner's identity +- **Volume access**: Governed by Unity Catalog permissions + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [databricks](#requirement\_databricks) | >= 1.40.0 | + +## Providers + +| Name | Version | +|------|---------| +| [databricks](#provider\_databricks) | 1.102.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [databricks_cluster.coding_assistants](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster) | resource | +| [databricks_file.init_script](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/file) | resource | +| [databricks_volume.init_scripts](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/volume) | resource | +| [databricks_current_user.me](https://registry.terraform.io/providers/databricks/databricks/latest/docs/data-sources/current_user) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [catalog\_name](#input\_catalog\_name) | Unity Catalog name for the volume | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | Name of the Databricks cluster | `string` | n/a | yes | +| [autotermination\_minutes](#input\_autotermination\_minutes) | Minutes of inactivity before cluster auto-terminates | `number` | `30` | no | +| [cluster\_mode](#input\_cluster\_mode) | Cluster mode: STANDARD or SINGLE\_NODE | `string` | `"STANDARD"` | no | +| [init\_script\_source\_path](#input\_init\_script\_source\_path) | Local path to the init script | `string` | `null` | no | +| [max\_workers](#input\_max\_workers) | Maximum number of workers for autoscaling | `number` | `3` | no | +| [min\_workers](#input\_min\_workers) | Minimum number of workers for autoscaling | `number` | `1` | no | +| [mlflow\_experiment\_name](#input\_mlflow\_experiment\_name) | MLflow experiment name for Claude Code tracing | `string` | `"/Workspace/Shared/claude-code-tracing"` | no | +| [node\_type\_id](#input\_node\_type\_id) | Node type for the cluster. Default is Standard_D8pds_v6 (modern, premium SSD + local NVMe). If unavailable in your region, consider Standard_DS13_v2 as fallback. | `string` | `"Standard_D8pds_v6"` | no | +| [num\_workers](#input\_num\_workers) | Number of worker nodes (null for autoscaling) | `number` | `null` | no | +| [schema\_name](#input\_schema\_name) | Schema name for the volume | `string` | `"default"` | no | +| [spark\_version](#input\_spark\_version) | Databricks Runtime version | `string` | `"17.3.x-cpu-ml-scala2.13"` | no | +| [tags](#input\_tags) | Custom tags for the cluster | `map(string)` | 
{
  "Environment": "dev",
  "Purpose": "coding-assistants"
}
| no | +| [volume\_name](#input\_volume\_name) | Volume name to store init scripts | `string` | `"coding_assistants"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | The ID of the created cluster | +| [cluster\_name](#output\_cluster\_name) | Name of the created cluster | +| [cluster\_url](#output\_cluster\_url) | URL to access the cluster in Databricks UI | +| [init\_script\_path](#output\_init\_script\_path) | Path to the init script in the volume | +| [mlflow\_experiment\_name](#output\_mlflow\_experiment\_name) | MLflow experiment name for tracing | +| [volume\_full\_name](#output\_volume\_full\_name) | Full name of the volume | +| [volume\_path](#output\_volume\_path) | Path to the volume containing init scripts | + + +## Post-Deployment Usage + +### On the Cluster + +After the cluster starts, users can: + +```bash +# Check installation status +check-claude + +# Debug Claude configuration +claude-debug + +# Use Claude Code +claude "Analyze the customer churn data" + +# Enable MLflow tracing +claude-tracing-enable + +# Check tracing status +claude-tracing-status +``` + +### Persistent Work Storage + +**IMPORTANT: Do not use Databricks Repos (`/Repos/...`) for active development work.** + +Databricks Repos folders can be unreliable for persistent storage and may lose uncommitted changes during cluster restarts or sync operations. Instead: + +✅ **Use `/Workspace/Users//` for all development work** + +This location provides reliable persistent storage across cluster restarts. Use the provided git helpers to manage version control: + +```bash +# Navigate to your workspace +cd /Workspace/Users/$(whoami)/ + +# Set up git (interactive helper) +git-workspace-init + +# Check git status and location +git-workspace-check + +# Configure git authentication +git-workspace-setup-auth +``` + +The git helpers will: +- Warn if you're working in `/Repos` (unreliable location) +- Help you clone existing repos or initialize new ones +- Check for uncommitted or unpushed changes +- Guide you through authentication setup (PAT, SSH, or credential helper) + +### Helper Commands + +The init script installs these helper commands in `~/.bashrc`: + +#### Claude CLI Commands + +| Command | Purpose | +|---------|---------| +| `check-claude` | Verify installation and configuration | +| `claude-debug` | Show detailed Claude CLI configuration | +| `claude-refresh-token` | Regenerate Claude settings | +| `claude-token-status` | Check token freshness and auto-refresh status | +| `claude-tracing-enable` | Enable MLflow tracing | +| `claude-tracing-status` | Check tracing status | +| `claude-tracing-disable` | Disable MLflow tracing | + +#### Git Workspace Commands + +| Command | Purpose | +|---------|---------| +| `git-workspace-init` | Interactive git setup in /Workspace (clone or init) | +| `git-workspace-check` | Check location and uncommitted/unpushed changes | +| `git-workspace-setup-auth` | Configure git authentication (PAT/SSH/credential helper) | + +#### VS Code/Cursor Remote Commands + +| Command | Purpose | +|---------|---------| +| `claude-vscode-setup` | Show Remote SSH setup guide | +| `claude-vscode-env` | Get Python interpreter path | +| `claude-vscode-check` | Verify Remote SSH configuration | +| `claude-vscode-config` | Generate settings.json snippet | + +## Cluster Access Modes + +### Single-User Access Mode + +```hcl +# Automatically configured by the module +data_security_mode = "SINGLE_USER" +single_user_name = data.databricks_current_user.me.user_name +``` + +### Standard Access Mode + +For standard access mode, you must: +1. Set up an allowlist for init scripts +2. Grant permissions to the volume + +See [Allowlist documentation](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/allowlist). + +## Troubleshooting + +### Init Script Fails + +Check cluster logs: +```bash +# Enable cluster log delivery in cluster config +# Then view: /cluster-logs//init_scripts/ +``` + +### Commands Not Found + +```bash +# Reload bashrc +source ~/.bashrc + +# Check PATH +echo $PATH | grep -E "(claude|opencode)" + +# Verify installation +check-coding-assistants +``` + +### Authentication Issues + +```bash +# Check environment variables +claude-debug + +# Verify token is set +echo $DATABRICKS_TOKEN + +# Regenerate configs +claude-refresh-token +opencode-refresh-config +``` + +### Script Size Limit + +Init scripts must be < 64KB. If exceeded: +- Break into multiple scripts +- Remove unnecessary comments +- Compress/optimize script + +## Security Considerations + +### Volume Permissions + +Ensure appropriate Unity Catalog permissions: + +```sql +-- Grant read access to volume +GRANT READ VOLUME ON VOLUME .. TO ; + +-- For standard access mode, add to allowlist +-- (Requires admin access) +``` + +### Token Security + +- Tokens are **never hardcoded** in configs +- Read from environment: `$DATABRICKS_TOKEN` +- Configs regenerate per session +- Settings files are user-readable only (`~/.claude/`, `~/.opencode/`) + +## Maintenance + +### Updating the Init Script + +1. Update the local init script file +2. Run `terraform apply` to upload new version +3. Restart clusters to apply changes + +```bash +terraform apply -target=module.coding_cluster.databricks_file.init_script +``` + +### Updating Cluster Configuration + +```bash +# Update variables in your config +# Then apply +terraform apply + +# Restart cluster for changes to take effect +``` + +## Cost Optimization + +- Use `autotermination_minutes` to automatically shut down idle clusters +- Use single-node mode for development: `cluster_mode = "SINGLE_NODE"` +- Enable autoscaling to scale down during low usage +- Consider spot instances (if supported by your cloud provider) + +## Limitations + +- Init scripts must be < 64KB +- Init script failures cause cluster launch to fail +- Requires Databricks Runtime 13.3 LTS+ for Unity Catalog volumes +- Standard access mode requires admin-configured allowlist + +## References + +- [Databricks Init Scripts Documentation](https://docs.databricks.com/init-scripts/) +- [Unity Catalog Volumes](https://docs.databricks.com/volumes/) +- [Databricks Terraform Provider](https://registry.terraform.io/providers/databricks/databricks/latest/docs) +- [Cluster Configuration](https://docs.databricks.com/compute/configure) + +## License + +This module is provided as-is for use with Databricks workspaces. + +## Contributing + +To contribute improvements to this module: +1. Test changes in an isolated Databricks workspace +2. Run `terraform validate` and `terraform fmt` +3. Update documentation for any new variables or outputs +4. Submit pull request with clear description of changes + +## Support + +For issues related to: +- **Module**: Open an issue in this repository +- **Init Script**: See the init script documentation +- **Databricks Platform**: Contact Databricks support +- **Claude/OpenCode**: Contact Anthropic or OpenCode support respectively diff --git a/modules/adb-coding-assistants-cluster/main.tf b/modules/adb-coding-assistants-cluster/main.tf new file mode 100644 index 00000000..578c1a20 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/main.tf @@ -0,0 +1,83 @@ +# Data source to get current user +data "databricks_current_user" "me" {} + +# Local value for init script path +locals { + init_script_path = var.init_script_source_path != null ? var.init_script_source_path : "${path.module}/scripts/install-claude.sh" +} + +# Create or reference the volume for init scripts +resource "databricks_volume" "init_scripts" { + name = var.volume_name + catalog_name = var.catalog_name + schema_name = var.schema_name + volume_type = "MANAGED" + comment = "Volume for Claude Code CLI init scripts" + + lifecycle { + ignore_changes = [owner] + } +} + +# Upload the init script to the volume +resource "databricks_file" "init_script" { + source = local.init_script_path + path = "${databricks_volume.init_scripts.volume_path}/install-claude.sh" +} + +# Create the cluster with init script +resource "databricks_cluster" "coding_assistants" { + cluster_name = var.cluster_name + spark_version = var.spark_version + node_type_id = var.node_type_id + autotermination_minutes = var.autotermination_minutes + data_security_mode = "SINGLE_USER" + single_user_name = data.databricks_current_user.me.user_name + + # Autoscaling or fixed size + # Autoscaling is not supported for single-node clusters + dynamic "autoscale" { + for_each = var.cluster_mode == "STANDARD" && var.num_workers == null ? [1] : [] + content { + min_workers = var.min_workers + max_workers = var.max_workers + } + } + + # For single-node clusters, num_workers must be 0 (driver-only) + # For standard clusters, use the provided num_workers value + num_workers = var.cluster_mode == "SINGLE_NODE" ? 0 : var.num_workers + + # Single node configuration + # According to Databricks docs: single-node clusters run Spark locally with no worker nodes + spark_conf = var.cluster_mode == "SINGLE_NODE" ? { + "spark.databricks.cluster.profile" = "singleNode" + "spark.master" = "local[*]" + } : {} + + custom_tags = merge( + var.tags, + { + "ManagedBy" = "Terraform" + }, + var.cluster_mode == "SINGLE_NODE" ? { + "ResourceClass" = "SingleNode" + } : {} + ) + + # Environment variables for Claude Code CLI + spark_env_vars = { + MLFLOW_EXPERIMENT_NAME = var.mlflow_experiment_name + } + + # Init script configuration + init_scripts { + volumes { + destination = "${databricks_volume.init_scripts.volume_path}/install-claude.sh" + } + } + + depends_on = [ + databricks_file.init_script + ] +} diff --git a/modules/adb-coding-assistants-cluster/outputs.tf b/modules/adb-coding-assistants-cluster/outputs.tf new file mode 100644 index 00000000..c19dca1a --- /dev/null +++ b/modules/adb-coding-assistants-cluster/outputs.tf @@ -0,0 +1,34 @@ +output "cluster_id" { + description = "The ID of the created cluster" + value = databricks_cluster.coding_assistants.id +} + +output "cluster_url" { + description = "URL to access the cluster in Databricks UI" + value = databricks_cluster.coding_assistants.url +} + +output "cluster_name" { + description = "Name of the created cluster" + value = databricks_cluster.coding_assistants.cluster_name +} + +output "volume_path" { + description = "Path to the volume containing init scripts" + value = databricks_volume.init_scripts.volume_path +} + +output "volume_full_name" { + description = "Full name of the volume" + value = "${var.catalog_name}.${var.schema_name}.${var.volume_name}" +} + +output "init_script_path" { + description = "Path to the init script in the volume" + value = databricks_file.init_script.path +} + +output "mlflow_experiment_name" { + description = "MLflow experiment name for tracing" + value = var.mlflow_experiment_name +} diff --git a/modules/adb-coding-assistants-cluster/scripts/README.md b/modules/adb-coding-assistants-cluster/scripts/README.md new file mode 100644 index 00000000..ae644a3a --- /dev/null +++ b/modules/adb-coding-assistants-cluster/scripts/README.md @@ -0,0 +1,353 @@ +# Claude Code CLI Installation Scripts + +This directory contains installation scripts for Claude Code CLI on Databricks clusters. + +## Scripts Overview + +| Script | Purpose | Network Required | +|--------|---------|------------------| +| `install-claude.sh` | Online installation (default) | ✅ Yes | + +> **Note**: For offline/air-gapped installations, use the separate [`adb-coding-assistants-cluster-offline`](../adb-coding-assistants-cluster-offline/README.md) module. + +## Quick Start + +### Online Installation (Default) + +For clusters with internet access: + +```hcl +resource "databricks_cluster" "claude_cluster" { + cluster_name = "claude-coding-assistant" + spark_version = data.databricks_spark_version.latest_lts.id + node_type_id = "Standard_D8pds_v6" + autotermination_minutes = 60 + num_workers = 0 + + init_scripts { + dbfs { + destination = "dbfs:/init-scripts/install-claude.sh" + } + } +} +``` + + +## What Gets Installed + +The script installs: + +- ✅ **Node.js 20.x** - Required runtime for Claude CLI +- ✅ **Claude Code CLI** - AI coding assistant +- ✅ **MLflow** - For tracing Claude interactions +- ✅ **System tools** - curl, wget, git, jq +- ✅ **Bash helpers** - Convenience functions for using Claude + +## Helper Commands + +After installation, these commands are available: + +```bash +# Verify installation +check-claude + +# Show debug info +claude-debug + +# Refresh authentication +claude-refresh-token + +# Token management +claude-token-status # Check token freshness +claude-setup-token-refresh # Enable automatic hourly refresh +claude-remove-token-refresh # Disable automatic refresh + +# Enable MLflow tracing +claude-tracing-enable + +# Check tracing status +claude-tracing-status + +# Disable tracing +claude-tracing-disable + +# VS Code/Cursor Remote SSH helpers +claude-vscode-setup # Show setup guide +claude-vscode-env # Get Python virtual environment path +claude-vscode-check # Verify VS Code/Cursor setup +claude-vscode-config # Generate VS Code settings.json snippet +``` + +## VS Code/Cursor Remote SSH Setup + +For remote development using VS Code or Cursor, follow these steps: + +### Quick Setup + +1. **Get Python interpreter path** (after SSH connection): + ```bash + claude-vscode-env + # Or manually: echo $DATABRICKS_VIRTUAL_ENV + ``` + +2. **Show complete setup guide**: + ```bash + claude-vscode-setup + ``` + +3. **Generate VS Code settings**: + ```bash + claude-vscode-config + ``` + +### Detailed Steps + +#### 1. Install Remote SSH Extension + +- **VS Code**: Install "Remote - SSH" extension from marketplace +- **Cursor**: Built-in Remote SSH extension (already included) + +#### 2. Configure Default Extensions + +Open Command Palette (`Cmd+Shift+P` / `Ctrl+Shift+P`): +- Type: `Remote-SSH: Settings` +- Or manually edit `settings.json`: + +```json +{ + "remote.SSH.defaultExtensions": [ + "ms-Python.python", + "ms-toolsai.jupyter" + ] +} +``` + +#### 3. Connect to Cluster + +- Command Palette → `Remote-SSH: Connect to Host` +- Enter your cluster SSH connection details + +#### 4. Select Python Interpreter + +After connecting: + +1. Run `claude-vscode-env` to get the Python path +2. Command Palette → `Python: Select Interpreter` +3. Enter or browse to: `/databricks/python*/pythonEnv-*/bin/python` + +**Important**: Always select the `pythonEnv-xxx` interpreter for full Databricks Runtime library access. + +#### 5. Verify Setup + +```bash +# Check setup status +claude-vscode-check + +# Test in a Python file +import pyspark +import pandas +import mlflow +print("Setup successful!") +``` + +### Important Notes + +- **IPYNB notebooks** and **`*.py` Databricks notebooks** have access to Databricks globals (`dbutils`, `spark`, etc.) +- **Regular Python `*.py` files** do NOT have access to Databricks globals +- Always select the `pythonEnv-xxx` interpreter for full Databricks Runtime library access + +### Standalone Helper Script + +A standalone helper script is also available: + +```bash +# Show setup guide +./scripts/vscode-setup.sh --guide + +# Get Python interpreter path +./scripts/vscode-setup.sh --env + +# Check current setup +./scripts/vscode-setup.sh --check + +# Generate settings.json +./scripts/vscode-setup.sh --settings +``` + +## Usage Examples + +```bash +# Interactive mode +claude + +# One-shot query +echo "Write a Python function to reverse a string" | claude --print + +# From file +claude < prompt.txt + +# With streaming +claude --stream < task.md +``` + +## Internet dependencies (online mode) + +The online installer requires access to: + +| Domain | Purpose | +|--------|---------| +| `claude.ai` | Claude CLI installer script | +| `storage.googleapis.com` | Claude CLI binaries (GCS bucket) | +| `deb.nodesource.com` | Node.js repository | +| `archive.ubuntu.com` | APT packages (x86_64) | +| `ports.ubuntu.com` | APT packages (ARM64) | +| `registry.npmjs.org` | NPM packages | +| `pypi.org` | Python package index | +| `files.pythonhosted.org` | Python package downloads | +| `raw.githubusercontent.com` | Databricks skills | +| `${DATABRICKS_HOST}` | Databricks API endpoints | + +> **Tip**: Run `./scripts/check-network-deps.sh` to verify all dependencies are accessible before installation. + +## Firewall configuration + +If using a firewall, allow HTTPS (443) and HTTP (80) to these domains, or use the offline installation method. + +## Environment Variables + +### Standard Variables (Set automatically by Databricks) + +- `DATABRICKS_HOST` - Workspace URL +- `DATABRICKS_TOKEN` - Authentication token + +### Optional Configuration + +- `MLFLOW_EXPERIMENT_NAME` - Custom experiment name (default: `/Workspace/Shared/claude-code-tracing`) + +## Architecture Support + +The installer supports: + +- ✅ **amd64** (x86_64) - Default +- ✅ **arm64** (aarch64) - Auto-detected + +## Network dependency checker + +Before installation, you can verify that all required domains are accessible using the network dependency checker: + +```bash +# Standard check +./scripts/check-network-deps.sh + +# Detailed output with HTTP status codes +./scripts/check-network-deps.sh --verbose +``` + +Example output: +``` +=== Claude Code Network Dependency Check === + +Checking required domains... + +[OK] claude.ai +[OK] storage.googleapis.com +[OK] deb.nodesource.com +[OK] archive.ubuntu.com +[OK] ports.ubuntu.com +[OK] registry.npmjs.org +[OK] pypi.org +[OK] files.pythonhosted.org +[OK] raw.githubusercontent.com + +---------------------------------------- +Result: 9/9 dependencies reachable + +SUCCESS: All dependencies are accessible +``` + +If any dependencies fail, the script provides troubleshooting guidance: +``` +[OK] claude.ai +[FAIL] deb.nodesource.com - Connection timed out +... +Result: 8/9 dependencies reachable + +FAILED: Some dependencies are not accessible + +Troubleshooting tips: + - Check firewall rules allow HTTPS (443) to the failed domains + - Verify proxy settings if behind a corporate proxy + - For air-gapped environments, use the offline installation module +``` + +## Troubleshooting + +### Installation fails during cluster startup + +Check the init script logs: +```bash +cat /tmp/init-script-claude.log +``` + +### Claude command not found + +Reload bashrc: +```bash +source ~/.bashrc +``` + +### Authentication errors + +Refresh token: +```bash +claude-refresh-token +``` + +### Installation works but Claude fails + +Check configuration: +```bash +check-claude +claude-debug +``` + +## File structure + +``` +scripts/ +├── install-claude.sh # Online installer +├── check-network-deps.sh # Network dependency checker +└── README.md # This file +``` + +> **Offline Installation**: See the [`adb-coding-assistants-cluster-offline`](../adb-coding-assistants-cluster-offline/README.md) module for offline/air-gapped installation support. + +## Version Compatibility + +- **Databricks Runtime**: 13.0+ LTS recommended +- **Python**: 3.9+ (included in DBR) +- **Node.js**: 20.x (installed by script) +- **MLflow**: 3.4+ (installed by script) + +## Security Notes + +### Authentication +- Uses Databricks personal access tokens (auto-configured) +- Tokens are ephemeral and cluster-scoped +- No long-lived credentials stored + +### Network Security +- All traffic uses HTTPS +- Authentication via `ANTHROPIC_AUTH_TOKEN` environment variable +- Custom headers for Databricks integration + + +## Support + +- **Claude CLI Issues**: [Claude AI Documentation](https://claude.ai/docs) +- **Databricks Issues**: Contact Databricks Support +- **Script Issues**: Open issue in repository + +## License + +See repository LICENSE file. diff --git a/modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh b/modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh new file mode 100755 index 00000000..3ef4bec2 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/scripts/check-network-deps.sh @@ -0,0 +1,238 @@ +#!/bin/bash +# +# Network Dependency Checker for Claude Code Installation +# +# Verifies connectivity to all required domains before running install-claude.sh. +# Run this script to diagnose network/firewall issues in restricted environments. +# +# Usage: +# ./check-network-deps.sh # Standard check +# ./check-network-deps.sh --verbose # Detailed output +# + +set -euo pipefail + +# ============================================================================ +# Configuration +# ============================================================================ + +CONNECT_TIMEOUT=5 +VERBOSE=false + +# Color codes (disabled if not a terminal) +if [[ -t 1 ]]; then + GREEN='\033[0;32m' + RED='\033[0;31m' + YELLOW='\033[0;33m' + BOLD='\033[1m' + NC='\033[0m' # No Color +else + GREEN='' + RED='' + YELLOW='' + BOLD='' + NC='' +fi + +# Dependencies to check: "domain|purpose|test_url" +DEPENDENCIES=( + "claude.ai|CLI installer script|https://claude.ai/install.sh" + "storage.googleapis.com|Claude CLI binaries|https://storage.googleapis.com/" + "deb.nodesource.com|Node.js repo|https://deb.nodesource.com/setup_20.x" + "archive.ubuntu.com|APT packages (x86)|http://archive.ubuntu.com/ubuntu/" + "ports.ubuntu.com|APT packages (ARM)|http://ports.ubuntu.com/ubuntu-ports/" + "registry.npmjs.org|NPM packages|https://registry.npmjs.org/" + "pypi.org|Python packages|https://pypi.org/simple/mlflow/" + "files.pythonhosted.org|Package downloads|https://files.pythonhosted.org/" + "raw.githubusercontent.com|Databricks skills|https://raw.githubusercontent.com/databricks-solutions/ai-dev-kit/main/README.md" +) + +# ============================================================================ +# Functions +# ============================================================================ + +usage() { + cat </dev/null; then + host "$domain" &>/dev/null + elif command -v nslookup &>/dev/null; then + nslookup "$domain" &>/dev/null + elif command -v getent &>/dev/null; then + getent hosts "$domain" &>/dev/null + else + # Fall back to ping for DNS resolution + ping -c 1 -W 2 "$domain" &>/dev/null + fi +} + +check_url() { + local url=$1 + local http_code + + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + --connect-timeout "$CONNECT_TIMEOUT" \ + --max-time $((CONNECT_TIMEOUT * 2)) \ + -L "$url" 2>/dev/null || echo "000") + + echo "$http_code" +} + +check_dependency() { + local entry=$1 + local domain purpose test_url + + IFS='|' read -r domain purpose test_url <<< "$entry" + + # Check DNS first + if ! check_dns "$domain"; then + log_fail "$domain - DNS resolution failed" + log_verbose "Purpose: $purpose" + log_verbose "Test URL: $test_url" + return 1 + fi + + # Check HTTP connectivity + local http_code + http_code=$(check_url "$test_url") + + if [[ "$http_code" =~ ^(2[0-9]{2}|3[0-9]{2})$ ]]; then + log_ok "$domain" + log_verbose "Purpose: $purpose" + log_verbose "HTTP status: $http_code" + log_verbose "Test URL: $test_url" + return 0 + else + case "$http_code" in + 000) + log_fail "$domain - Connection timed out" + ;; + 400) + # 400 is common for API endpoints at root - domain is reachable + log_ok "$domain" + log_verbose "Purpose: $purpose" + log_verbose "HTTP status: $http_code (API endpoint - root returns 400)" + log_verbose "Test URL: $test_url" + return 0 + ;; + 403) + log_fail "$domain - Access forbidden (HTTP 403)" + ;; + 404) + # 404 means domain is reachable, just URL changed + log_ok "$domain" + log_verbose "Purpose: $purpose" + log_verbose "HTTP status: $http_code (domain reachable)" + log_verbose "Test URL: $test_url" + return 0 + ;; + *) + log_fail "$domain - HTTP $http_code" + ;; + esac + log_verbose "Purpose: $purpose" + log_verbose "Test URL: $test_url" + return 1 + fi +} + +# ============================================================================ +# Main +# ============================================================================ + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --verbose|-v) + VERBOSE=true + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +# Check for curl +if ! command -v curl &>/dev/null; then + echo "Error: curl is required but not installed" + exit 1 +fi + +echo "" +echo -e "${BOLD}=== Claude Code Network Dependency Check ===${NC}" +echo "" +echo "Checking required domains..." +echo "" + +pass_count=0 +fail_count=0 +total=${#DEPENDENCIES[@]} + +for dep in "${DEPENDENCIES[@]}"; do + if check_dependency "$dep"; then + ((pass_count++)) + else + ((fail_count++)) + fi +done + +echo "" +echo "----------------------------------------" +echo -e "Result: ${BOLD}${pass_count}/${total}${NC} dependencies reachable" + +if [[ $fail_count -gt 0 ]]; then + echo "" + echo -e "${RED}FAILED: Some dependencies are not accessible${NC}" + echo "" + echo "Troubleshooting tips:" + echo " - Check firewall rules allow HTTPS (443) to the failed domains" + echo " - Verify proxy settings if behind a corporate proxy" + echo " - For air-gapped environments, use the offline installation module" + echo "" + exit 1 +else + echo "" + echo -e "${GREEN}SUCCESS: All dependencies are accessible${NC}" + echo "" + exit 0 +fi diff --git a/modules/adb-coding-assistants-cluster/scripts/install-claude-minimal.sh b/modules/adb-coding-assistants-cluster/scripts/install-claude-minimal.sh new file mode 100755 index 00000000..5679d84e --- /dev/null +++ b/modules/adb-coding-assistants-cluster/scripts/install-claude-minimal.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Databricks Cluster Init Script - Claude Code CLI (Minimal Version) +# Installs Claude Code CLI with basic configuration only +# + +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +LOG_FILE="/tmp/init-script-claude.log" +log() { + echo "[$(date '+%H:%M:%S')] $1" | tee -a "$LOG_FILE" +} + +# Install system dependencies +log "Installing system dependencies..." +sudo apt-get update -qq -y >> "$LOG_FILE" 2>&1 +sudo apt-get install -y -qq curl git >> "$LOG_FILE" 2>&1 || log "Warning: Some packages failed to install" + +# Install Node.js 20.x +if ! command -v node >/dev/null 2>&1; then + log "Installing Node.js 20.x..." + curl -fsSL --max-time 300 --retry 3 https://deb.nodesource.com/setup_20.x | sudo -E bash - >> "$LOG_FILE" 2>&1 + sudo apt-get install -y -qq nodejs >> "$LOG_FILE" 2>&1 + log "Node.js installed: $(node --version)" +else + log "Node.js already installed: $(node --version)" +fi + +# Install Claude Code CLI +if ! command -v claude >/dev/null 2>&1; then + log "Installing Claude Code CLI..." + curl -fsSL https://claude.ai/install.sh | bash >> "$LOG_FILE" 2>&1 + log "Claude Code CLI installed" +else + log "Claude Code CLI already installed" +fi + +# Add basic configuration to bashrc +log "Configuring bashrc..." + +# Remove old Claude section if it exists +if [ -f "$HOME/.bashrc" ]; then + sed -i '/### CLAUDE_CODE_MINIMAL_START ###/,/### CLAUDE_CODE_MINIMAL_END ###/d' "$HOME/.bashrc" || true +fi + +# Add Claude to PATH and set environment variables +cat >> "$HOME/.bashrc" <<'BASHRC_EOF' + +### CLAUDE_CODE_MINIMAL_START ### +# Claude Code CLI - Minimal Setup +export PATH="$HOME/.claude/bin:$HOME/.local/bin:$PATH" + +# Set Anthropic environment variables for Claude CLI +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + export ANTHROPIC_AUTH_TOKEN="$DATABRICKS_TOKEN" + export ANTHROPIC_BASE_URL="${DATABRICKS_HOST}/serving-endpoints/anthropic" + export ANTHROPIC_MODEL="databricks-claude-sonnet-4-5" + export ANTHROPIC_CUSTOM_HEADERS="x-databricks-disable-beta-headers: true" + export CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 +fi +### CLAUDE_CODE_MINIMAL_END ### +BASHRC_EOF + +log "Configuration complete. Log file: $LOG_FILE" +log "After cluster starts, run: source ~/.bashrc" diff --git a/modules/adb-coding-assistants-cluster/scripts/install-claude.sh b/modules/adb-coding-assistants-cluster/scripts/install-claude.sh new file mode 100755 index 00000000..52891fc2 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/scripts/install-claude.sh @@ -0,0 +1,760 @@ +#!/bin/bash +# +# Databricks Cluster Init Script - Claude Code CLI +# Installs Claude Code CLI with MLflow tracing +# +# Note: For offline/air-gapped installations, use the adb-coding-assistants-cluster-offline module instead +# + +set -uo pipefail +export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a CI=true + +L="/tmp/init-script-claude.log" +log() { echo "[$(date '+%H:%M:%S')] $1" | tee -a "$L"; } +cmd_exists() { command -v "$1" >/dev/null 2>&1; } + +# Install Claude Code CLI +install_claude() { + if cmd_exists claude; then + log "[OK] Claude Code already installed" + return 0 + fi + + log "Installing Claude Code CLI..." + if curl -fsSL https://claude.ai/install.sh | bash &>>$L; then + log "[OK] Claude Code installation completed" + return 0 + else + log "[WARN] Claude Code installation failed (will be available after manual install)" + return 1 + fi +} + +# Install Node.js (required for Claude Code CLI) +install_nodejs() { + if cmd_exists node && cmd_exists npm; then + log "[OK] Node.js already installed ($(node --version))" + return 0 + fi + + log "Installing Node.js 20.x..." + if curl -fsSL --max-time 300 --retry 3 https://deb.nodesource.com/setup_20.x | sudo -E bash - &>>$L; then + if sudo apt-get update -qq -y &>>$L && sudo apt-get install -y -qq nodejs &>>$L; then + if cmd_exists node && cmd_exists npm; then + log "[OK] Node.js/npm installed successfully ($(node --version))" + return 0 + fi + fi + fi + + log "[WARN] Node.js installation failed (Claude Code CLI will not work)" + return 1 +} + +# Add helper functions to bashrc +setup_bashrc() { + local START_MARKER="### CLAUDE_CODE_HELPERS_START ###" + local END_MARKER="### CLAUDE_CODE_HELPERS_END ###" + + # Backup bashrc + [ -f "$HOME/.bashrc" ] && cp "$HOME/.bashrc" "$HOME/.bashrc.backup-$(date +%s)" + + # Remove any existing Claude sections (between markers) + if [ -f "$HOME/.bashrc" ]; then + if grep -q "$START_MARKER" "$HOME/.bashrc" 2>/dev/null; then + log "Removing old bashrc helpers..." + # Remove everything between START and END markers (inclusive) + sed -i "/$START_MARKER/,/$END_MARKER/d" "$HOME/.bashrc" + fi + fi + + W="${DATABRICKS_HOST}" + E="${MLFLOW_EXPERIMENT_NAME:-/Workspace/Shared/claude-code-tracing}" + + log "Adding helpers to bashrc..." + + cat >> "$HOME/.bashrc" <<'EOF' + +### CLAUDE_CODE_HELPERS_START ### +# Claude Code CLI Setup (auto-generated - do not edit manually) +export PATH="$HOME/.claude/bin:$HOME/.local/bin:$PATH" + +# Claude Code MLflow tracing helpers +export DATABRICKS_HOST="${DATABRICKS_HOST:-WS_PH}" +export MLFLOW_EXPERIMENT_NAME="${MLFLOW_EXPERIMENT_NAME:-EXP_PH}" + +# Set Anthropic environment variables for Claude CLI +# NOTE: These env vars are the PRIMARY authentication method and take precedence +# over settings.json. They are always fresh because they're set on every login. +# The settings.json file serves as a fallback for cases where env vars aren't set. +# Using ANTHROPIC_AUTH_TOKEN only (not ANTHROPIC_API_KEY) to avoid auth conflicts. +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + export ANTHROPIC_AUTH_TOKEN="$DATABRICKS_TOKEN" + export ANTHROPIC_BASE_URL="${DATABRICKS_HOST}/serving-endpoints/anthropic" + export ANTHROPIC_MODEL="databricks-claude-sonnet-4-5" + export ANTHROPIC_CUSTOM_HEADERS="x-databricks-disable-beta-headers: true" + export CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 +fi + +# Internal function to generate Claude settings (single source of truth) +_generate_claude_config() { + local config_file="$HOME/.claude/settings.json" + + cat > "$config_file" </dev/null 2>&1; then + if ! jq empty "$config_file" 2>/dev/null; then + echo "[WARN] Claude settings JSON validation failed" >&2 + return 1 + fi + fi + + # Store token hash for change detection + if [ -n "$DATABRICKS_TOKEN" ]; then + echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 > "$HOME/.claude/.token_hash" 2>/dev/null || true + fi + + return 0 +} + +# Check if token has changed and refresh if needed +_check_and_refresh_token() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + return 0 # Skip if token not available + fi + + local config_file="$HOME/.claude/settings.json" + local token_hash_file="$HOME/.claude/.token_hash" + + # Calculate current token hash + local current_hash + current_hash=$(echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 2>/dev/null || echo "") + + if [ -z "$current_hash" ]; then + return 0 # Skip if hash calculation failed + fi + + # Check if token has changed + if [ -f "$token_hash_file" ]; then + local stored_hash + stored_hash=$(cat "$token_hash_file" 2>/dev/null || echo "") + if [ "$current_hash" = "$stored_hash" ]; then + return 0 # Token unchanged, no refresh needed + fi + fi + + # Token changed or first time - refresh config + mkdir -p "$HOME/.claude" + if _generate_claude_config >/dev/null 2>&1; then + # Only show message if in interactive shell (not cron) + if [ -t 0 ]; then + echo "[OK] Claude Code token refreshed automatically" + fi + return 0 + fi + + return 1 +} + +# Auto-generate Claude settings from environment on first login +# NOTE: settings.json acts as a FALLBACK - env vars (set above) are the primary method. +# This is only generated if the file doesn't exist, to provide authentication when +# env vars might not be present (e.g., in some non-standard shell environments). +if [ ! -f "$HOME/.claude/settings.json" ] && [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + mkdir -p "$HOME/.claude" + if _generate_claude_config; then + echo "[OK] Claude Code settings.json created (fallback - env vars take precedence)" + else + echo "[WARN] Failed to generate Claude settings (run claude-refresh-token to retry)" + fi +fi + +# Auto-refresh token on shell login if it has changed +# This ensures settings.json stays in sync with DATABRICKS_TOKEN +if [ -n "$DATABRICKS_TOKEN" ] && [ -n "$DATABRICKS_HOST" ]; then + _check_and_refresh_token +fi + +# Regenerate Claude settings from current environment +claude-refresh-token() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + echo "[WARN] DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + echo " On Databricks clusters, these should be automatically available" + return 1 + fi + + mkdir -p "$HOME/.claude" + if _generate_claude_config; then + echo "[OK] Claude Code settings updated with:" + echo " DATABRICKS_HOST: $DATABRICKS_HOST" + echo " DATABRICKS_TOKEN: ${DATABRICKS_TOKEN:0:20}..." + else + echo "[WARN] Failed to update Claude settings" + return 1 + fi +} + +# Setup cron job for periodic token refresh (runs hourly) +claude-setup-token-refresh() { + local cron_cmd="[ -n \"\$DATABRICKS_TOKEN\" ] && [ -n \"\$DATABRICKS_HOST\" ] && source \"\$HOME/.bashrc\" && _check_and_refresh_token >/dev/null 2>&1" + local cron_job="0 * * * * $cron_cmd" + local cron_file="$HOME/.claude/token-refresh-cron" + + # Create cron wrapper script + mkdir -p "$HOME/.claude" + cat > "$cron_file" <<'CRON_SCRIPT' +#!/bin/bash +# Auto-generated cron script for Claude token refresh +# This script is called by cron to refresh the Claude token periodically + +# Source bashrc to get functions +if [ -f "$HOME/.bashrc" ]; then + source "$HOME/.bashrc" >/dev/null 2>&1 +fi + +# Check and refresh token if needed +_check_and_refresh_token +CRON_SCRIPT + chmod +x "$cron_file" + + # Check if cron job already exists + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + echo "[OK] Token refresh cron job already configured" + return 0 + fi + + # Add cron job + (crontab -l 2>/dev/null; echo "0 * * * * $cron_file") | crontab - + if [ $? -eq 0 ]; then + echo "[OK] Token refresh cron job configured (runs hourly)" + echo " To remove: crontab -e" + else + echo "[WARN] Failed to setup cron job (may require cron service)" + return 1 + fi +} + +# Remove token refresh cron job +claude-remove-token-refresh() { + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + crontab -l 2>/dev/null | grep -v "token-refresh-cron" | crontab - + echo "[OK] Token refresh cron job removed" + else + echo "[INFO] No token refresh cron job found" + fi +} + +# Check token freshness status +claude-token-status() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + echo "[WARN] DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + return 1 + fi + + local config_file="$HOME/.claude/settings.json" + local token_hash_file="$HOME/.claude/.token_hash" + + echo "=== Claude Token Status ===" + echo "" + + # Check if config file exists + if [ -f "$config_file" ]; then + echo "[OK] Settings file: $config_file" + local file_age + file_age=$(stat -c %Y "$config_file" 2>/dev/null || stat -f %m "$config_file" 2>/dev/null || echo "0") + local current_time + current_time=$(date +%s) + local age_hours + age_hours=$(( (current_time - file_age) / 3600 )) + echo " Last updated: ${age_hours} hour(s) ago" + else + echo "[ERROR] Settings file: missing" + fi + + echo "" + + # Check token hash + if [ -f "$token_hash_file" ]; then + local current_hash + current_hash=$(echo -n "$DATABRICKS_TOKEN" | sha256sum | cut -d' ' -f1 2>/dev/null || echo "") + local stored_hash + stored_hash=$(cat "$token_hash_file" 2>/dev/null || echo "") + if [ "$current_hash" = "$stored_hash" ] && [ -n "$current_hash" ]; then + echo "[OK] Token: matches stored hash (up to date)" + else + echo "[WARN] Token: differs from stored hash (needs refresh)" + echo " Run: claude-refresh-token" + fi + else + echo "[INFO] Token hash: not stored (will be created on next refresh)" + fi + + echo "" + + # Check cron job + if crontab -l 2>/dev/null | grep -q "token-refresh-cron"; then + echo "[OK] Auto-refresh: enabled (hourly cron job)" + else + echo "[INFO] Auto-refresh: disabled" + echo " Enable with: claude-setup-token-refresh" + fi +} + +claude-tracing-enable() { + if [ -z "$DATABRICKS_TOKEN" ] || [ -z "$DATABRICKS_HOST" ]; then + echo "[WARN] DATABRICKS_TOKEN and DATABRICKS_HOST must be set" + echo " On Databricks clusters, these should be automatically available" + return 1 + fi + + if ! command -v mlflow >/dev/null 2>&1; then + echo "[WARN] MLflow is not installed" + return 1 + fi + + # Create experiment if it doesn't exist + python3 </dev/null 2>&1; then + echo "[OK] Claude Code CLI: $(which claude)" + claude --version 2>&1 | head -1 || echo " (version check failed)" + else + echo "[ERROR] Claude Code CLI: not found" + [ -f "$HOME/.claude/bin/claude" ] && echo " Binary exists at: $HOME/.claude/bin/claude" + [ -f "$HOME/.local/bin/claude" ] && echo " Binary exists at: $HOME/.local/bin/claude" + fi + echo "" + + # Check configs + echo "Configuration files:" + if [ -f "$HOME/.claude/settings.json" ]; then + echo " [OK] Claude settings: $HOME/.claude/settings.json" + echo " Preview: $(head -3 $HOME/.claude/settings.json | tail -1)" + else + echo " [ERROR] Claude settings: missing" + fi + echo "" + + # Check environment + echo "Environment variables:" + [ -n "$DATABRICKS_HOST" ] && echo " [OK] DATABRICKS_HOST: ${DATABRICKS_HOST}" || echo " [ERROR] DATABRICKS_HOST: not set" + [ -n "$DATABRICKS_TOKEN" ] && echo " [OK] DATABRICKS_TOKEN: ${DATABRICKS_TOKEN:0:20}..." || echo " [ERROR] DATABRICKS_TOKEN: not set" + [ -n "$ANTHROPIC_API_KEY" ] && echo " [OK] ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:0:20}..." || echo " [ERROR] ANTHROPIC_API_KEY: not set" + [ -n "$ANTHROPIC_AUTH_TOKEN" ] && echo " [OK] ANTHROPIC_AUTH_TOKEN: ${ANTHROPIC_AUTH_TOKEN:0:20}..." || echo " [ERROR] ANTHROPIC_AUTH_TOKEN: not set" + [ -n "$ANTHROPIC_BASE_URL" ] && echo " [OK] ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL}" || echo " [ERROR] ANTHROPIC_BASE_URL: not set" + [ -n "$ANTHROPIC_MODEL" ] && echo " [OK] ANTHROPIC_MODEL: ${ANTHROPIC_MODEL}" || echo " [ERROR] ANTHROPIC_MODEL: not set" + [ -n "$ANTHROPIC_CUSTOM_HEADERS" ] && echo " [OK] ANTHROPIC_CUSTOM_HEADERS: ${ANTHROPIC_CUSTOM_HEADERS}" || echo " [ERROR] ANTHROPIC_CUSTOM_HEADERS: not set" + echo "" + + # Check MLflow + if command -v mlflow >/dev/null 2>&1; then + echo "[OK] MLflow: $(mlflow --version 2>&1)" + else + echo "[ERROR] MLflow: not found" + fi + echo "" + + # Test Claude authentication + echo "Testing Claude CLI authentication:" + if command -v claude >/dev/null 2>&1; then + if [ -n "$ANTHROPIC_API_KEY" ] || [ -n "$ANTHROPIC_AUTH_TOKEN" ]; then + echo " [OK] Authentication configured via environment variables" + echo " Test with: echo 'what is 1+1?' | claude --print" + else + echo " [WARN] ANTHROPIC_API_KEY/ANTHROPIC_AUTH_TOKEN not set" + echo " Run: source ~/.bashrc" + fi + fi + echo "" + + # VS Code/Cursor Remote SSH info + echo "VS Code/Cursor Remote SSH:" + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo " [OK] Python virtual environment: $venv_path" + echo " Run 'claude-vscode-setup' for setup instructions" + else + echo " [INFO] Run 'claude-vscode-setup' for Remote SSH setup guide" + fi + echo "" + + echo "Run 'source ~/.bashrc' if commands are still not found" +} + +claude-debug() { + echo "=== Claude CLI Debug Info ===" + echo "" + echo "Settings file:" + [ -f "$HOME/.claude/settings.json" ] && cat "$HOME/.claude/settings.json" || echo " Missing!" + echo "" + echo "Environment:" + env | grep -E "ANTHROPIC|DATABRICKS" || echo " No relevant env vars" + echo "" + echo "Claude config directory:" + ls -la "$HOME/.claude/" 2>/dev/null || echo " Directory doesn't exist" +} + +# VS Code/Cursor Remote SSH helpers +claude-vscode-env() { + # Show the Databricks virtual environment path for VS Code/Cursor + if [ -n "$DATABRICKS_VIRTUAL_ENV" ]; then + echo "$DATABRICKS_VIRTUAL_ENV" + else + # Try to find pythonEnv-* directories + local python_envs + python_envs=$(find /databricks/python* -maxdepth 1 -type d -name "pythonEnv-*" 2>/dev/null | head -1) + if [ -n "$python_envs" ]; then + echo "$python_envs" + else + echo "[WARN] DATABRICKS_VIRTUAL_ENV not set and pythonEnv-* not found" + echo " Try: echo \$DATABRICKS_VIRTUAL_ENV" + return 1 + fi + fi +} + +claude-vscode-setup() { + echo "=== VS Code/Cursor Remote SSH Setup Guide ===" + echo "" + echo "1. Install Remote SSH Extension" + echo " - VS Code: Install 'Remote - SSH' extension" + echo " - Cursor: Built-in Remote SSH extension (already included)" + echo "" + echo "2. Configure Default Extensions" + echo " Open Command Palette (Cmd+Shift+P / Ctrl+Shift+P):" + echo " -> Remote-SSH: Settings" + echo "" + echo " Or edit settings.json and add:" + echo "" + cat <<'VSCODE_SETTINGS' + "remote.SSH.defaultExtensions": [ + "ms-Python.python", + "ms-toolsai.jupyter" + ] +VSCODE_SETTINGS + echo "" + echo "3. Connect to Cluster" + echo " - Command Palette -> Remote-SSH: Connect to Host" + echo " - Enter your cluster SSH connection details" + echo "" + echo "4. Select Python Interpreter" + echo " After connecting, run this command to get the Python path:" + echo "" + echo " $ claude-vscode-env" + echo "" + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo " Current virtual environment:" + echo " $venv_path" + echo "" + echo " Then in VS Code/Cursor:" + echo " - Command Palette -> Python: Select Interpreter" + echo " - Paste the path above or browse to it" + else + echo " Run 'echo \$DATABRICKS_VIRTUAL_ENV' to find the path" + fi + echo "" + echo "5. Important Notes" + echo " * IPYNB notebooks and *.py Databricks notebooks have access to" + echo " Databricks globals (dbutils, spark, etc.)" + echo " * Regular Python *.py files do NOT have access to Databricks globals" + echo " * Always select the pythonEnv-xxx interpreter for full Databricks" + echo " Runtime library access" + echo "" + echo "6. Verify Setup" + echo " Run: claude-vscode-check" +} + +claude-vscode-check() { + echo "=== VS Code/Cursor Remote SSH Setup Check ===" + echo "" + + # Check for virtual environment + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "[OK] Python Virtual Environment:" + echo " $venv_path" + if [ -d "$venv_path/bin" ]; then + echo " [OK] Virtual environment directory exists" + if [ -f "$venv_path/bin/python" ]; then + echo " [OK] Python executable found" + echo " Python version: $($venv_path/bin/python --version 2>&1 || echo 'unknown')" + else + echo " [WARN] Python executable not found" + fi + else + echo " [WARN] Virtual environment directory not found" + fi + else + echo "[ERROR] Python Virtual Environment: Not found" + echo " Run: echo \$DATABRICKS_VIRTUAL_ENV" + fi + echo "" + + # Check for Python + if command -v python3 >/dev/null 2>&1; then + echo "[OK] Python3 available: $(which python3)" + echo " Version: $(python3 --version 2>&1)" + else + echo "[ERROR] Python3 not found in PATH" + fi + echo "" + + # Check for Databricks runtime libraries + echo "Databricks Runtime Libraries:" + python3 <<'PYTHON_CHECK' +import sys +libraries = ['pyspark', 'pandas', 'numpy', 'mlflow', 'databricks'] +found = [] +missing = [] + +for lib in libraries: + try: + __import__(lib) + found.append(lib) + except ImportError: + missing.append(lib) + +if found: + print(f" [OK] Available: {', '.join(found)}") +if missing: + print(f" [WARN] Missing: {', '.join(missing)}") + +# Check for Databricks globals (only available in notebooks) +try: + import dbutils + print(" [OK] dbutils available (notebook context)") +except: + print(" [INFO] dbutils not available (normal for .py files)") +PYTHON_CHECK + + echo "" + echo "VS Code/Cursor Configuration:" + echo " Run 'claude-vscode-setup' for setup instructions" + echo " Run 'claude-vscode-env' to get Python interpreter path" +} + +claude-vscode-config() { + # Generate VS Code settings.json snippet + local venv_path + venv_path=$(claude-vscode-env 2>/dev/null) + + echo "=== VS Code/Cursor settings.json Configuration ===" + echo "" + echo "Add this to your VS Code/Cursor settings.json:" + echo "" + echo "{" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "," + echo " \"python.defaultInterpreterPath\": \"$venv_path/bin/python\"" + fi + echo "}" + echo "" + if [ $? -eq 0 ] && [ -n "$venv_path" ]; then + echo "Python interpreter path:" + echo " $venv_path/bin/python" + echo "" + echo "To set this in VS Code/Cursor:" + echo " 1. Command Palette -> Python: Select Interpreter" + echo " 2. Enter interpreter path: $venv_path/bin/python" + else + echo "To find Python interpreter path, run:" + echo " claude-vscode-env" + fi +} +### CLAUDE_CODE_HELPERS_END ### +EOF + + sed -i "s|WS_PH|$W|g; s|EXP_PH|$E|g" "$HOME/.bashrc" + log "[OK] Bashrc helpers added" + log " Experiment: $E" +} + +# Install Databricks skills for Claude Code +install_databricks_skills() { + local skills_dir="$HOME/.claude/skills" + local repo_url="https://raw.githubusercontent.com/databricks-solutions/ai-dev-kit/main/databricks-skills" + + # Core skills to install (curated list for most common use cases) + local core_skills=( + "databricks-config" + "databricks-python-sdk" + "databricks-unity-catalog" + "databricks-jobs" + "asset-bundles" + "databricks-app-python" + "model-serving" + "mlflow-evaluation" + "aibi-dashboards" + "spark-declarative-pipelines" + ) + + log "Installing Databricks skills for Claude Code..." + + # Create skills directory + mkdir -p "$skills_dir" + + local installed=0 + local failed=0 + + for skill in "${core_skills[@]}"; do + local skill_dir="$skills_dir/$skill" + + # Skip if already exists + if [ -d "$skill_dir" ] && [ -f "$skill_dir/SKILL.md" ]; then + log " [INFO] Skill '$skill' already installed" + installed=$((installed + 1)) + continue + fi + + # Create skill directory + mkdir -p "$skill_dir" + + # Download SKILL.md (required) + if curl -sSL -f "${repo_url}/${skill}/SKILL.md" -o "$skill_dir/SKILL.md" 2>>$L; then + log " [OK] Installed skill: $skill" + installed=$((installed + 1)) + else + log " [WARN] Failed to download skill: $skill" + rm -rf "$skill_dir" + failed=$((failed + 1)) + fi + done + + if [ $installed -gt 0 ]; then + log "[OK] Databricks skills installed: $installed skills" + [ $failed -gt 0 ] && log "[WARN] Failed to install: $failed skills" + return 0 + else + log "[WARN] No Databricks skills installed" + return 1 + fi +} + +# Main installation +main() { + log "Starting installation..." + + # Install system dependencies (curl, git, jq - commonly used by Claude Code) + log "Installing system dependencies..." + if sudo apt-get update -qq -y &>>$L; then + if sudo apt-get install -y -qq curl git jq &>>$L; then + log "[OK] System dependencies installed (curl, git, jq)" + else + log "[WARN] Some system dependencies failed to install" + fi + else + log "[WARN] apt-get update failed" + fi + + # Install MLflow with Databricks support + log "Installing MLflow with Databricks support..." + if pip install --quiet --upgrade "mlflow[databricks]>=3.4" &>>$L; then + log "[OK] MLflow installed successfully" + else + log "[WARN] MLflow installation failed (tracing features will not work)" + fi + + # Install tools (continue even if some fail) + install_nodejs || log "[WARN] Node.js installation skipped or failed" + install_claude || log "[WARN] Claude Code installation skipped or failed" + + # Install Databricks skills for Claude Code + install_databricks_skills || log "[WARN] Databricks skills installation incomplete" + + # Configure tools + if setup_bashrc; then + log "[OK] Bashrc configuration completed" + else + log "[WARN] Bashrc configuration failed" + fi + + log "" + log "=== Installation Summary ===" + log "Installation complete. Full log: $L" + log "" + log "Installed components:" + log " - Claude Code CLI" + log " - Node.js runtime" + log " - MLflow with Databricks support" + log " - Databricks skills (patterns and best practices)" + log "" + log "Next steps (on cluster login):" + log " 1. Run: source ~/.bashrc" + log " 2. Verify: check-claude" + log " 3. Use: claude command" + log "" + log "Databricks skills installed in: ~/.claude/skills/" + log "Skills available: databricks-config, python-sdk, unity-catalog," + log " jobs, asset-bundles, apps, model-serving, mlflow, dashboards, pipelines" + log "" + log "Helper commands:" + log " - check-claude: Verify installation status" + log " - claude-debug: Show Claude CLI configuration details" + log " - claude-refresh-token: Regenerate Claude settings" + log " - claude-token-status: Check token freshness and auto-refresh status" + log " - claude-setup-token-refresh: Enable hourly automatic token refresh (optional)" + log " - claude-remove-token-refresh: Disable automatic token refresh" + log " - claude-tracing-enable/disable/status: Manage MLflow tracing" + log " - claude-vscode-setup: Show VS Code/Cursor Remote SSH setup guide" + log " - claude-vscode-env: Get Python virtual environment path" + log " - claude-vscode-check: Verify VS Code/Cursor setup" + log " - claude-vscode-config: Generate VS Code settings.json snippet" + return 0 +} + +main +exit 0 diff --git a/modules/adb-coding-assistants-cluster/scripts/vscode-setup.sh b/modules/adb-coding-assistants-cluster/scripts/vscode-setup.sh new file mode 100755 index 00000000..89b21ada --- /dev/null +++ b/modules/adb-coding-assistants-cluster/scripts/vscode-setup.sh @@ -0,0 +1,248 @@ +#!/bin/bash +# +# VS Code/Cursor Remote SSH Setup Helper for Databricks Clusters +# This script helps configure VS Code or Cursor for remote development on Databricks clusters +# + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +log_info() { echo -e "${BLUE}ℹ${NC} $1"; } +log_success() { echo -e "${GREEN}✓${NC} $1"; } +log_warning() { echo -e "${YELLOW}⚠${NC} $1"; } +log_error() { echo -e "${RED}✗${NC} $1"; } + +# Find Databricks Python virtual environment +find_python_env() { + if [ -n "${DATABRICKS_VIRTUAL_ENV:-}" ]; then + echo "$DATABRICKS_VIRTUAL_ENV" + return 0 + fi + + # Try to find pythonEnv-* directories + local python_envs + python_envs=$(find /databricks/python* -maxdepth 1 -type d -name "pythonEnv-*" 2>/dev/null | head -1) + if [ -n "$python_envs" ]; then + echo "$python_envs" + return 0 + fi + + return 1 +} + +# Show setup instructions +show_setup_guide() { + echo "==========================================" + echo "VS Code/Cursor Remote SSH Setup Guide" + echo "==========================================" + echo "" + + echo "1. Install Remote SSH Extension" + echo " • VS Code: Install 'Remote - SSH' extension from marketplace" + echo " • Cursor: Built-in Remote SSH extension (already included)" + echo "" + + echo "2. Configure Default Extensions" + echo " Open Command Palette (Cmd+Shift+P / Ctrl+Shift+P):" + echo " → Type: Remote-SSH: Settings" + echo "" + echo " Or manually edit settings.json:" + echo "" + echo " {" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + echo " }" + echo "" + + echo "3. Connect to Cluster" + echo " • Command Palette → Remote-SSH: Connect to Host" + echo " • Enter your cluster SSH connection details" + echo " • Format: user@hostname or use SSH config entry" + echo "" + + echo "4. Select Python Interpreter" + local venv_path + if venv_path=$(find_python_env 2>/dev/null); then + echo " ✓ Found Python virtual environment:" + echo " $venv_path" + echo "" + echo " In VS Code/Cursor:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Enter interpreter path:" + echo " $venv_path/bin/python" + echo "" + echo " Or copy this path:" + echo " $venv_path/bin/python" + else + echo " ⚠ Could not auto-detect Python virtual environment" + echo " Run this command to find it:" + echo " echo \$DATABRICKS_VIRTUAL_ENV" + echo "" + echo " Then in VS Code/Cursor:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Paste the path from above" + fi + echo "" + + echo "5. Important Notes" + echo " • IPYNB notebooks and *.py Databricks notebooks have access to" + echo " Databricks globals (dbutils, spark, etc.)" + echo " • Regular Python *.py files do NOT have access to Databricks globals" + echo " • Always select the pythonEnv-xxx interpreter for full Databricks" + echo " Runtime library access (pyspark, pandas, numpy, mlflow, etc.)" + echo "" + + echo "6. Verify Setup" + echo " After connecting, verify Python interpreter:" + echo " • Command Palette → Python: Select Interpreter" + echo " • Should show: pythonEnv-xxx/bin/python" + echo "" + echo " Test in a Python file:" + echo " import pyspark" + echo " import pandas" + echo " print('Setup successful!')" +} + +# Generate VS Code settings.json snippet +generate_settings() { + local venv_path + venv_path=$(find_python_env 2>/dev/null || echo "") + + echo "{" + echo " \"remote.SSH.defaultExtensions\": [" + echo " \"ms-Python.python\"," + echo " \"ms-toolsai.jupyter\"" + echo " ]" + if [ -n "$venv_path" ]; then + echo "," + echo " \"python.defaultInterpreterPath\": \"$venv_path/bin/python\"" + fi + echo "}" +} + +# Check current setup +check_setup() { + echo "==========================================" + echo "VS Code/Cursor Setup Check" + echo "==========================================" + echo "" + + # Check for virtual environment + local venv_path + if venv_path=$(find_python_env 2>/dev/null); then + log_success "Python Virtual Environment found:" + echo " $venv_path" + + if [ -d "$venv_path/bin" ]; then + log_success "Virtual environment directory exists" + if [ -f "$venv_path/bin/python" ]; then + log_success "Python executable found" + echo " Python version: $($venv_path/bin/python --version 2>&1 || echo 'unknown')" + else + log_warning "Python executable not found" + fi + else + log_warning "Virtual environment directory not found" + fi + else + log_error "Python Virtual Environment not found" + echo " Run: echo \$DATABRICKS_VIRTUAL_ENV" + fi + echo "" + + # Check for Python + if command -v python3 >/dev/null 2>&1; then + log_success "Python3 available: $(which python3)" + echo " Version: $(python3 --version 2>&1)" + else + log_error "Python3 not found in PATH" + fi + echo "" + + # Check for Databricks runtime libraries + echo "Databricks Runtime Libraries:" + python3 <<'PYTHON_CHECK' +import sys +libraries = ['pyspark', 'pandas', 'numpy', 'mlflow'] +found = [] +missing = [] + +for lib in libraries: + try: + __import__(lib) + found.append(lib) + except ImportError: + missing.append(lib) + +if found: + print(f" ✓ Available: {', '.join(found)}") +if missing: + print(f" ⚠ Missing: {', '.join(missing)}") + +# Check for Databricks globals (only available in notebooks) +try: + import dbutils + print(" ✓ dbutils available (notebook context)") +except: + print(" ℹ dbutils not available (normal for .py files)") +PYTHON_CHECK + + echo "" + echo "Next steps:" + echo " • Run this script with --guide to see setup instructions" + echo " • Run this script with --settings to generate settings.json" +} + +# Main +main() { + case "${1:-}" in + --guide|-g) + show_setup_guide + ;; + --settings|-s) + generate_settings + ;; + --check|-c) + check_setup + ;; + --env|-e) + find_python_env || { + log_error "Could not find Python virtual environment" + echo "Try: echo \$DATABRICKS_VIRTUAL_ENV" + exit 1 + } + ;; + --help|-h|"") + echo "VS Code/Cursor Remote SSH Setup Helper" + echo "" + echo "Usage: $0 [OPTION]" + echo "" + echo "Options:" + echo " --guide, -g Show complete setup guide" + echo " --settings, -s Generate VS Code settings.json snippet" + echo " --check, -c Check current setup status" + echo " --env, -e Show Python virtual environment path" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 --guide # Show setup instructions" + echo " $0 --env # Get Python interpreter path" + echo " $0 --check # Verify setup" + ;; + *) + log_error "Unknown option: $1" + echo "Run '$0 --help' for usage information" + exit 1 + ;; + esac +} + +main "$@" diff --git a/modules/adb-coding-assistants-cluster/variables.tf b/modules/adb-coding-assistants-cluster/variables.tf new file mode 100644 index 00000000..cb751f24 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/variables.tf @@ -0,0 +1,94 @@ +variable "cluster_name" { + description = "Name of the Databricks cluster" + type = string +} + +variable "catalog_name" { + description = "Unity Catalog catalog name for the volume. The metastore must have a root storage credential configured." + type = string +} + +variable "schema_name" { + description = "Schema name for the volume" + type = string + default = "default" +} + +variable "volume_name" { + description = "Volume name to store init scripts" + type = string + default = "coding_assistants" +} + +variable "init_script_source_path" { + description = "Local path to the init script" + type = string + default = null +} + +variable "spark_version" { + description = "Databricks Runtime version" + type = string + default = "17.3.x-cpu-ml-scala2.13" +} + +variable "node_type_id" { + description = "Node type for the cluster. Default is Standard_D8pds_v6 (modern, premium SSD + local NVMe). If unavailable in your region, consider Standard_DS13_v2 as fallback." + type = string + default = "Standard_D4ds_v5" +} + +variable "autotermination_minutes" { + description = "Minutes of inactivity before cluster auto-terminates" + type = number + default = 30 +} + +variable "num_workers" { + description = "Number of worker nodes (null for autoscaling). For SINGLE_NODE clusters, this is automatically set to 0." + type = number + default = null +} + +variable "min_workers" { + description = "Minimum number of workers for autoscaling" + type = number + default = 1 +} + +variable "max_workers" { + description = "Maximum number of workers for autoscaling" + type = number + default = 3 +} + +variable "mlflow_experiment_name" { + description = "MLflow experiment name for Claude Code tracing" + type = string + default = "/Workspace/Shared/claude-code-tracing" +} + +variable "cluster_mode" { + description = <<-EOT + Cluster mode: STANDARD or SINGLE_NODE. + - STANDARD: Multi-node cluster with worker nodes (supports autoscaling) + - SINGLE_NODE: Single-node cluster with no worker nodes (driver-only, runs Spark locally). + For SINGLE_NODE clusters, num_workers is automatically set to 0 and autoscaling is disabled. + EOT + type = string + default = "STANDARD" + + validation { + condition = contains(["STANDARD", "SINGLE_NODE"], var.cluster_mode) + error_message = "cluster_mode must be either STANDARD or SINGLE_NODE" + } +} + +variable "tags" { + description = "Custom tags for the cluster" + type = map(string) + default = { + Environment = "dev" + Purpose = "coding-assistants" + } +} diff --git a/modules/adb-coding-assistants-cluster/versions.tf b/modules/adb-coding-assistants-cluster/versions.tf new file mode 100644 index 00000000..07223296 --- /dev/null +++ b/modules/adb-coding-assistants-cluster/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + databricks = { + source = "databricks/databricks" + version = ">= 1.40.0" + } + } +} diff --git a/modules/aws-uc-external-location-file-events/Makefile b/modules/aws-uc-external-location-file-events/Makefile new file mode 100644 index 00000000..653039d8 --- /dev/null +++ b/modules/aws-uc-external-location-file-events/Makefile @@ -0,0 +1,7 @@ +.PHONY: docs test_docs + +docs: + terraform-docs -c ../../.terraform-docs.yml . + +test_docs: + terraform-docs -c ../../.terraform-docs.yml --output-check . diff --git a/modules/aws-uc-external-location-file-events/README.md b/modules/aws-uc-external-location-file-events/README.md new file mode 100644 index 00000000..58027040 --- /dev/null +++ b/modules/aws-uc-external-location-file-events/README.md @@ -0,0 +1,155 @@ +# aws-uc-external-location-file-events + +Creates AWS Unity Catalog **storage credentials** and **external locations** with +[managed file events](https://docs.databricks.com/aws/en/connect/unity-catalog/cloud-storage/manage-external-locations) +enabled via Amazon SNS + SQS (managed SQS). The module builds the +**self-assuming IAM role** and the IAM policy that Unity Catalog requires, including the +extra `sns:*` / `sqs:*` / S3 bucket-notification permissions needed for automatic file-event +setup. + +This is useful for [file arrival triggers](https://docs.databricks.com/aws/en/jobs/file-arrival-triggers) +and other ingestion patterns (for example [Auto Loader](https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/file-events-explained)) +that benefit from cloud storage change notifications. + +This is the AWS counterpart to +[`adb-uc-external-location-file-events`](../adb-uc-external-location-file-events) (Azure). + +## Prerequisites + +- Unity Catalog–enabled Databricks workspace whose assigned metastore is the target metastore + (the Databricks provider should authenticate to that workspace), **or** an account-level + provider with `metastore_id` set. +- One or more existing S3 buckets that back the external locations. This module does **not** + create the buckets — the caller owns them (see the example). +- Permission to create IAM roles/policies in the AWS account that owns the bucket(s). +- Permission to create UC storage credentials and external locations (metastore admin or the + `CREATE STORAGE CREDENTIAL` / `CREATE EXTERNAL LOCATION` privileges). + +## What it creates + +| Resource | Purpose | +| -------- | ------- | +| `aws_iam_policy` | S3 data access on the target bucket(s), `sts:AssumeRole` self-permission, and (when `enable_file_events`) the `csms-*`-scoped SNS/SQS/bucket-notification permissions | +| `aws_iam_role` | Self-assuming role trusted by the UC master role (gated by the Databricks account ID as external ID) | +| `time_sleep` | Waits for IAM propagation before UC validates the credential (avoids transient "non self-assuming" / 403 errors) | +| `databricks_storage_credential` | UC storage credential backed by the IAM role | +| `databricks_external_location` | One per entry in `external_locations`, with automatic managed SQS file events | +| `databricks_grants` | Optional UC grants on the credential and locations | + +## Managed file events (automatic mode) + +When `enable_file_events = true` (default), the IAM policy includes the documented +`ManagedFileEventsSetupStatement` / `ManagedFileEventsListStatement` / +`ManagedFileEventsTeardownStatement` statements, scoped to the target bucket(s) and the +`csms-*` SNS/SQS namespace. Unity Catalog then provisions one SNS topic and one SQS queue +(prefixed `csms-*`) per external location and configures the S3 bucket notification. Without +these permissions, file events silently fail to provision even though the location reports it +as enabled. + +## Example + +```hcl +# The caller owns the bucket. +resource "aws_s3_bucket" "demo" { + bucket = "my-uc-demo-bucket" + force_destroy = true +} + +module "uc_locations" { + source = "../../modules/aws-uc-external-location-file-events" + + name_prefix = "demo" + databricks_account_id = "00000000-0000-0000-0000-000000000000" + bucket_names = [aws_s3_bucket.demo.id] + + external_locations = [ + { + name = "demo_landing" + url = "s3://${aws_s3_bucket.demo.id}/landing" + comment = "Landing zone with managed SQS file events" + } + ] + + location_grants = [ + { + principal = "data-engineers" + privileges = ["BROWSE", "READ_FILES", "WRITE_FILES", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME"] + } + ] +} +``` + +See also [examples/aws-uc-external-location-file-events](../../examples/aws-uc-external-location-file-events). + + +## Requirements + +| Name | Version | +| ---- | ------- | +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | >= 5.0 | +| [databricks](#requirement\_databricks) | >= 1.81.1 | +| [time](#requirement\_time) | >= 0.9.0 | + +## Providers + +| Name | Version | +| ---- | ------- | +| [aws](#provider\_aws) | >= 5.0 | +| [databricks](#provider\_databricks) | >= 1.81.1 | +| [time](#provider\_time) | >= 0.9.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| ---- | ---- | +| [aws_iam_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [databricks_external_location.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/external_location) | resource | +| [databricks_grants.credential](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource | +| [databricks_grants.location](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource | +| [databricks_storage_credential.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/storage_credential) | resource | +| [time_sleep.wait_iam](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| [bucket\_names](#input\_bucket\_names) | Names of the S3 buckets that the storage credential IAM role is granted access to (and, when file events are enabled, allowed to configure notifications on). The buckets must already exist; this module does not create them. | `list(string)` | n/a | yes | +| [databricks\_account\_id](#input\_databricks\_account\_id) | Databricks account ID. Used as the sts:ExternalId in the IAM role trust policy (the Unity Catalog storage credential external ID). | `string` | n/a | yes | +| [external\_locations](#input\_external\_locations) | External locations to create. Each location has managed SQS file events enabled when enable\_file\_events is true. | 
list(object({
    name      = string
    url       = string
    comment   = optional(string, "Managed by Terraform")
    read_only = optional(bool, false)
  }))
| n/a | yes | +| [name\_prefix](#input\_name\_prefix) | Prefix used to derive default names for the IAM role, IAM policy, and storage credential. | `string` | n/a | yes | +| [create\_storage\_credential](#input\_create\_storage\_credential) | When true, create a storage credential backed by the IAM role this module manages. When false, reuse existing\_credential\_name (the IAM role/policy are still managed). | `bool` | `true` | no | +| [credential\_grants](#input\_credential\_grants) | UC grants applied to the storage credential. Defaults to empty (owner-only). | 
list(object({
    principal  = string
    privileges = list(string)
  }))
| `[]` | no | +| [enable\_file\_events](#input\_enable\_file\_events) | Enable automatic managed file events (SNS topic + SQS queue + S3 bucket notification, csms-* prefixed) on every external location. Adds the required sns/sqs/bucket-notification permissions to the IAM policy. | `bool` | `true` | no | +| [existing\_credential\_name](#input\_existing\_credential\_name) | Name of an existing storage credential to reference on the external locations when create\_storage\_credential is false. | `string` | `""` | no | +| [force\_destroy](#input\_force\_destroy) | Force destroy the storage credential and external locations even if dependents exist. | `bool` | `false` | no | +| [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy attached to the role. Defaults to "-policy". | `string` | `""` | no | +| [iam\_propagation\_delay](#input\_iam\_propagation\_delay) | Delay to wait after creating the IAM role/policy before creating the storage credential and external locations, so IAM changes propagate (avoids "non self-assuming" / 403 validation errors). Set to "" to disable the wait (e.g. when the role already exists). | `string` | `"60s"` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name of the IAM role used by the storage credential. Defaults to "-uc". | `string` | `""` | no | +| [location\_grants](#input\_location\_grants) | UC grants applied to every external location.
Recommended privileges for data engineers: BROWSE, READ\_FILES, WRITE\_FILES,
CREATE\_EXTERNAL\_TABLE, CREATE\_EXTERNAL\_VOLUME. | 
list(object({
    principal  = string
    privileges = list(string)
  }))
| `[]` | no | +| [storage\_credential\_comment](#input\_storage\_credential\_comment) | Comment applied to the created storage credential. | `string` | `"Storage credential for external locations with file events. Managed by Terraform."` | no | +| [storage\_credential\_name](#input\_storage\_credential\_name) | Name for the created storage credential. Defaults to "-storage-credential". | `string` | `""` | no | +| [tags](#input\_tags) | Tags applied to the IAM role and policy. | `map(string)` | `{}` | no | +| [uc\_master\_role\_arn](#input\_uc\_master\_role\_arn) | ARN of the Unity Catalog AWS master role that assumes the storage credential role. Defaults to the Databricks commercial (non-GovCloud) UC master role. | `string` | `"arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL"` | no | + +## Outputs + +| Name | Description | +| ---- | ----------- | +| [external\_location\_ids](#output\_external\_location\_ids) | Map of external location name to ID | +| [external\_location\_names](#output\_external\_location\_names) | Names of the created external locations | +| [external\_location\_urls](#output\_external\_location\_urls) | Map of external location name to URL | +| [file\_events\_enabled](#output\_file\_events\_enabled) | Whether managed file events are enabled on the external locations | +| [iam\_policy\_arn](#output\_iam\_policy\_arn) | ARN of the IAM policy attached to the role | +| [iam\_role\_arn](#output\_iam\_role\_arn) | ARN of the IAM role trusted by the Unity Catalog storage credential | +| [iam\_role\_name](#output\_iam\_role\_name) | Name of the IAM role backing the storage credential | +| [storage\_credential\_id](#output\_storage\_credential\_id) | ID of the created storage credential (null when reusing an existing credential) | +| [storage\_credential\_name](#output\_storage\_credential\_name) | Name of the storage credential used by the external locations | + diff --git a/modules/aws-uc-external-location-file-events/main.tf b/modules/aws-uc-external-location-file-events/main.tf new file mode 100644 index 00000000..a27febf2 --- /dev/null +++ b/modules/aws-uc-external-location-file-events/main.tf @@ -0,0 +1,220 @@ +data "aws_caller_identity" "current" {} + +locals { + iam_role_name = var.iam_role_name != "" ? var.iam_role_name : "${var.name_prefix}-uc" + iam_policy_name = var.iam_policy_name != "" ? var.iam_policy_name : "${local.iam_role_name}-policy" + storage_credential_name = var.storage_credential_name != "" ? var.storage_credential_name : "${var.name_prefix}-storage-credential" + + role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${local.iam_role_name}" + credential_name = var.create_storage_credential ? databricks_storage_credential.this[0].name : var.existing_credential_name + + bucket_arns = [for b in var.bucket_names : "arn:aws:s3:::${b}"] + bucket_object_arns = [for b in var.bucket_names : "arn:aws:s3:::${b}/*"] + + # Base data-access statements (always present). + base_statements = [ + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + "s3:PutObjectAcl", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetBucketLocation" + ] + Resource = concat(local.bucket_arns, local.bucket_object_arns) + }, + { + Sid = "AllowSelfAssume" + Effect = "Allow" + Action = ["sts:AssumeRole"] + Resource = [local.role_arn] + } + ] + + # Managed file events (Automatic mode): lets Unity Catalog configure S3 bucket + # notifications, create the SNS topic + SQS queue (csms-* prefixed) and subscribe + # the queue to the topic. Scoped to the target buckets and the csms-* namespace. + # Docs: https://docs.databricks.com/aws/en/connect/unity-catalog/cloud-storage/manage-external-locations + file_event_statements = var.enable_file_events ? [ + { + Sid = "ManagedFileEventsSetupStatement" + Effect = "Allow" + Action = [ + "s3:GetBucketNotification", + "s3:PutBucketNotification", + "sns:ListSubscriptionsByTopic", + "sns:GetTopicAttributes", + "sns:SetTopicAttributes", + "sns:CreateTopic", + "sns:TagResource", + "sns:Publish", + "sns:Subscribe", + "sqs:CreateQueue", + "sqs:DeleteMessage", + "sqs:ReceiveMessage", + "sqs:SendMessage", + "sqs:GetQueueUrl", + "sqs:GetQueueAttributes", + "sqs:SetQueueAttributes", + "sqs:TagQueue", + "sqs:ChangeMessageVisibility", + "sqs:PurgeQueue" + ] + Resource = concat(local.bucket_arns, ["arn:aws:sqs:*:*:csms-*", "arn:aws:sns:*:*:csms-*"]) + }, + { + Sid = "ManagedFileEventsListStatement" + Effect = "Allow" + Action = ["sqs:ListQueues", "sqs:ListQueueTags", "sns:ListTopics"] + Resource = ["arn:aws:sqs:*:*:csms-*", "arn:aws:sns:*:*:csms-*"] + }, + { + Sid = "ManagedFileEventsTeardownStatement" + Effect = "Allow" + Action = ["sns:Unsubscribe", "sns:DeleteTopic", "sqs:DeleteQueue"] + Resource = ["arn:aws:sqs:*:*:csms-*", "arn:aws:sns:*:*:csms-*"] + } + ] : [] +} + +# Self-assuming role trust policy required by Unity Catalog storage credentials: +# 1. The UC master role may assume this role, gated by the external ID. +# 2. The role may assume itself (paired with the AllowSelfAssume permission below). +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + type = "AWS" + identifiers = [var.uc_master_role_arn] + } + condition { + test = "StringEquals" + variable = "sts:ExternalId" + values = [var.databricks_account_id] + } + } + + statement { + sid = "ExplicitSelfRoleAssumption" + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + } + condition { + test = "ArnLike" + variable = "aws:PrincipalArn" + values = [local.role_arn] + } + } +} + +resource "aws_iam_policy" "this" { + name = local.iam_policy_name + + policy = jsonencode({ + Version = "2012-10-17" + Id = "${var.bucket_names[0]}-access" + Statement = concat(local.base_statements, local.file_event_statements) + }) + + tags = merge(var.tags, { + Name = local.iam_policy_name + }) +} + +resource "aws_iam_role" "this" { + name = local.iam_role_name + assume_role_policy = data.aws_iam_policy_document.assume_role.json + managed_policy_arns = [aws_iam_policy.this.arn] + + tags = merge(var.tags, { + Name = local.iam_role_name + }) +} + +# Give the IAM role/policy time to propagate before Unity Catalog validates the +# credential and external locations (avoids transient "non self-assuming" / 403s). +resource "time_sleep" "wait_iam" { + count = var.iam_propagation_delay == "" ? 0 : 1 + + create_duration = var.iam_propagation_delay + + triggers = { + role_arn = aws_iam_role.this.arn + policy_arn = aws_iam_policy.this.arn + } +} + +resource "databricks_storage_credential" "this" { + count = var.create_storage_credential ? 1 : 0 + + name = local.storage_credential_name + comment = var.storage_credential_comment + + aws_iam_role { + role_arn = aws_iam_role.this.arn + } + + force_destroy = var.force_destroy ? true : null + + depends_on = [time_sleep.wait_iam] +} + +resource "databricks_external_location" "this" { + for_each = { for loc in var.external_locations : loc.name => loc } + + name = each.value.name + url = each.value.url + credential_name = local.credential_name + comment = each.value.comment + read_only = each.value.read_only + + enable_file_events = var.enable_file_events + dynamic "file_event_queue" { + for_each = var.enable_file_events ? [1] : [] + content { + managed_sqs {} + } + } + + force_destroy = var.force_destroy ? true : null + + depends_on = [ + databricks_storage_credential.this, + time_sleep.wait_iam, + ] +} + +resource "databricks_grants" "credential" { + count = var.create_storage_credential && length(var.credential_grants) > 0 ? 1 : 0 + + storage_credential = databricks_storage_credential.this[0].id + + dynamic "grant" { + for_each = var.credential_grants + content { + principal = grant.value.principal + privileges = grant.value.privileges + } + } +} + +resource "databricks_grants" "location" { + for_each = length(var.location_grants) > 0 ? databricks_external_location.this : {} + + external_location = each.value.id + + dynamic "grant" { + for_each = var.location_grants + content { + principal = grant.value.principal + privileges = grant.value.privileges + } + } +} diff --git a/modules/aws-uc-external-location-file-events/outputs.tf b/modules/aws-uc-external-location-file-events/outputs.tf new file mode 100644 index 00000000..98f7ae4e --- /dev/null +++ b/modules/aws-uc-external-location-file-events/outputs.tf @@ -0,0 +1,44 @@ +output "iam_role_name" { + description = "Name of the IAM role backing the storage credential" + value = aws_iam_role.this.name +} + +output "iam_role_arn" { + description = "ARN of the IAM role trusted by the Unity Catalog storage credential" + value = aws_iam_role.this.arn +} + +output "iam_policy_arn" { + description = "ARN of the IAM policy attached to the role" + value = aws_iam_policy.this.arn +} + +output "storage_credential_name" { + description = "Name of the storage credential used by the external locations" + value = local.credential_name +} + +output "storage_credential_id" { + description = "ID of the created storage credential (null when reusing an existing credential)" + value = try(databricks_storage_credential.this[0].id, null) +} + +output "external_location_names" { + description = "Names of the created external locations" + value = [for loc in databricks_external_location.this : loc.name] +} + +output "external_location_ids" { + description = "Map of external location name to ID" + value = { for k, v in databricks_external_location.this : k => v.id } +} + +output "external_location_urls" { + description = "Map of external location name to URL" + value = { for k, v in databricks_external_location.this : k => v.url } +} + +output "file_events_enabled" { + description = "Whether managed file events are enabled on the external locations" + value = var.enable_file_events +} diff --git a/modules/aws-uc-external-location-file-events/providers.tf b/modules/aws-uc-external-location-file-events/providers.tf new file mode 100644 index 00000000..dced1992 --- /dev/null +++ b/modules/aws-uc-external-location-file-events/providers.tf @@ -0,0 +1,18 @@ +# This module expects: +# - an AWS provider with permission to manage IAM roles/policies for the account +# that owns the S3 bucket(s) backing the external location(s) +# - a Databricks provider authenticated to a UC-enabled workspace whose assigned +# metastore is the one you want the storage credential / external locations in +# (workspace-level auth), or an account-level provider with metastore_id set on +# the resources. +# +# Example: +# +# provider "aws" { +# region = "ap-southeast-2" +# profile = "my-aws-profile" +# } +# +# provider "databricks" { +# profile = "my-workspace-profile" +# } diff --git a/modules/aws-uc-external-location-file-events/variables.tf b/modules/aws-uc-external-location-file-events/variables.tf new file mode 100644 index 00000000..d48668b8 --- /dev/null +++ b/modules/aws-uc-external-location-file-events/variables.tf @@ -0,0 +1,127 @@ +variable "name_prefix" { + type = string + description = "Prefix used to derive default names for the IAM role, IAM policy, and storage credential." +} + +variable "databricks_account_id" { + type = string + description = "Databricks account ID. Used as the sts:ExternalId in the IAM role trust policy (the Unity Catalog storage credential external ID)." +} + +variable "uc_master_role_arn" { + type = string + description = "ARN of the Unity Catalog AWS master role that assumes the storage credential role. Defaults to the Databricks commercial (non-GovCloud) UC master role." + default = "arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL" +} + +variable "bucket_names" { + type = list(string) + description = "Names of the S3 buckets that the storage credential IAM role is granted access to (and, when file events are enabled, allowed to configure notifications on). The buckets must already exist; this module does not create them." + + validation { + condition = length(var.bucket_names) > 0 + error_message = "At least one bucket name is required." + } +} + +variable "external_locations" { + type = list(object({ + name = string + url = string + comment = optional(string, "Managed by Terraform") + read_only = optional(bool, false) + })) + description = "External locations to create. Each location has managed SQS file events enabled when enable_file_events is true." + + validation { + condition = length(var.external_locations) > 0 + error_message = "At least one external location is required." + } +} + +variable "enable_file_events" { + type = bool + description = "Enable automatic managed file events (SNS topic + SQS queue + S3 bucket notification, csms-* prefixed) on every external location. Adds the required sns/sqs/bucket-notification permissions to the IAM policy." + default = true +} + +variable "create_storage_credential" { + type = bool + description = "When true, create a storage credential backed by the IAM role this module manages. When false, reuse existing_credential_name (the IAM role/policy are still managed)." + default = true +} + +variable "existing_credential_name" { + type = string + description = "Name of an existing storage credential to reference on the external locations when create_storage_credential is false." + default = "" + + validation { + condition = var.create_storage_credential || var.existing_credential_name != "" + error_message = "existing_credential_name must be set when create_storage_credential is false." + } +} + +variable "storage_credential_name" { + type = string + description = "Name for the created storage credential. Defaults to \"-storage-credential\"." + default = "" +} + +variable "storage_credential_comment" { + type = string + description = "Comment applied to the created storage credential." + default = "Storage credential for external locations with file events. Managed by Terraform." +} + +variable "iam_role_name" { + type = string + description = "Name of the IAM role used by the storage credential. Defaults to \"-uc\"." + default = "" +} + +variable "iam_policy_name" { + type = string + description = "Name of the IAM policy attached to the role. Defaults to \"-policy\"." + default = "" +} + +variable "iam_propagation_delay" { + type = string + description = "Delay to wait after creating the IAM role/policy before creating the storage credential and external locations, so IAM changes propagate (avoids \"non self-assuming\" / 403 validation errors). Set to \"\" to disable the wait (e.g. when the role already exists)." + default = "60s" +} + +variable "force_destroy" { + type = bool + description = "Force destroy the storage credential and external locations even if dependents exist." + default = false +} + +variable "credential_grants" { + type = list(object({ + principal = string + privileges = list(string) + })) + description = "UC grants applied to the storage credential. Defaults to empty (owner-only)." + default = [] +} + +variable "location_grants" { + type = list(object({ + principal = string + privileges = list(string) + })) + description = <<-EOT + UC grants applied to every external location. + Recommended privileges for data engineers: BROWSE, READ_FILES, WRITE_FILES, + CREATE_EXTERNAL_TABLE, CREATE_EXTERNAL_VOLUME. + EOT + default = [] +} + +variable "tags" { + type = map(string) + description = "Tags applied to the IAM role and policy." + default = {} +} diff --git a/modules/aws-uc-external-location-file-events/versions.tf b/modules/aws-uc-external-location-file-events/versions.tf new file mode 100644 index 00000000..23eeccad --- /dev/null +++ b/modules/aws-uc-external-location-file-events/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + databricks = { + source = "databricks/databricks" + version = ">= 1.81.1" + } + time = { + source = "hashicorp/time" + version = ">= 0.9.0" + } + } +}