From 0cb37fe65c3fbc87fdf58c87b508b50254ce0589 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Tue, 18 Aug 2026 15:03:50 -0400 Subject: [PATCH 1/2] Update flowx documentation --- .build-constraints.txt | 16 +- docs/app/docs/[[...slug]]/page.tsx | 3 +- docs/app/global.css | 2 +- .../docs/{options.mdx => configuration.mdx} | 15 +- docs/content/docs/guide.mdx | 60 +++--- docs/content/docs/index.mdx | 14 +- docs/content/docs/installation.mdx | 171 ++++++++---------- docs/content/docs/meta.json | 2 +- docs/mdx-components.tsx | 24 +++ docs/source.config.ts | 10 +- 10 files changed, 177 insertions(+), 140 deletions(-) rename docs/content/docs/{options.mdx => configuration.mdx} (94%) create mode 100644 docs/mdx-components.tsx diff --git a/.build-constraints.txt b/.build-constraints.txt index 3a6ff79..2ad60de 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -1,9 +1,9 @@ -hatchling==1.31.0 \ - --hash=sha256:6b48ad4068a482ed7239b3a8215bc55b47aad3345d58dfc94e553c5d2d46211b \ - --hash=sha256:aac80bec8b6fe35e8480f1c335be8910fa210a0e6f735a139be205dadcacb544 -packaging==26.2 \ - --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ - --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 +hatchling==1.32.0 \ + --hash=sha256:0bdbde4a52b06c37e3eca395f85a762bf0ef06fe374fd8ae429dc6be10230f5f \ + --hash=sha256:0e17c9c3b9aa7c625acc8d0f5b622f107d5049af9ecf5ada4de1aada5be7cdbc +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c # via hatchling pathspec==1.1.1 \ --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ @@ -13,6 +13,10 @@ pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 # via hatchling +tomlkit==0.15.1 \ + --hash=sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304 \ + --hash=sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97 + # via hatchling trove-classifiers==2026.6.1.19 \ --hash=sha256:ab4c4ec93cc4a4e7815fa759906e05e6bb3f2fbd92ea0f897288c6a43efd15b3 \ --hash=sha256:c5132b4b61a829d11cfbd2d72e97f20a45ed6edb95e45c5efdeb5e00836b2745 diff --git a/docs/app/docs/[[...slug]]/page.tsx b/docs/app/docs/[[...slug]]/page.tsx index 40a24cf..e15409b 100644 --- a/docs/app/docs/[[...slug]]/page.tsx +++ b/docs/app/docs/[[...slug]]/page.tsx @@ -1,6 +1,7 @@ import { source } from '@/lib/source'; import { DocsPage, DocsBody, DocsTitle, DocsDescription } from 'fumadocs-ui/page'; import { notFound } from 'next/navigation'; +import { getMDXComponents } from '@/mdx-components'; export default async function Page(props: { params: Promise<{ slug?: string[] }>; @@ -16,7 +17,7 @@ export default async function Page(props: { {page.data.title} {page.data.description} - + ); diff --git a/docs/app/global.css b/docs/app/global.css index 7408c0a..44885b1 100644 --- a/docs/app/global.css +++ b/docs/app/global.css @@ -1,5 +1,5 @@ @import 'tailwindcss'; -@import 'fumadocs-ui/css/neutral.css'; +@import 'fumadocs-ui/css/solar.css'; @import 'fumadocs-ui/css/preset.css'; @source '../node_modules/fumadocs-ui/dist/**/*.js'; diff --git a/docs/content/docs/options.mdx b/docs/content/docs/configuration.mdx similarity index 94% rename from docs/content/docs/options.mdx rename to docs/content/docs/configuration.mdx index 05e1649..b3437ce 100644 --- a/docs/content/docs/options.mdx +++ b/docs/content/docs/configuration.mdx @@ -1,6 +1,6 @@ --- -title: Translation options -description: Control flowx's translation behavior and outputs +title: Configuration +description: Control flowx's translation behavior and outputs. --- import { Callout } from 'fumadocs-ui/components/callout'; @@ -130,6 +130,17 @@ The weighted score is `sum(activity weights) + #datasets + #linked_services + #c - control-flow / parameter-setting (ForEach/If/Switch/SetVariable/AppendVariable/Filter/Wait/Until) = **2** - all other activities (Copy/Web/Lookup/etc.) = **3** (hardest) +## Translation report + +During translation, flowx writes a transient `translation_report.json` under +`/.work/` (default `./flowx_output/.work/`). It lists every activity, its +translation strategy, warnings raised during translation, and the location of any generated +artifacts. Review the translation report for any warnings, unsupported resources, or to-do +items before deploying to your Databricks workspace. + +The `package` phase prunes `.work/` after building the bundle. Pass `--keep-intermediates` to +retain it. + ## Coverage results table & dashboard (Genie Code) When running with workspace auth (Genie Code, or a configured Databricks profile) the `package` diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx index d758899..2572aa5 100644 --- a/docs/content/docs/guide.mdx +++ b/docs/content/docs/guide.mdx @@ -7,14 +7,17 @@ import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; import { Callout } from 'fumadocs-ui/components/callout'; import { Steps, Step } from 'fumadocs-ui/components/steps'; -This guide walks through a complete migration: install the flowx skills in your agentic tool, hand it a directory of Azure Data Factory JSON exports, and end up with a Databricks Asset Bundle you can deploy. +This guide walks through an end-to-end conversion of an Azure Data Factory pipeline to Lakeflow Jobs. You will install the +flowx skills in your agentic tool, hand it a directory of Azure Data Factory JSON exports, and end up with a Databricks +Asset Bundle you can deploy. ## Export pipeline templates as JSON -In the Azure Data Factory portal, open *Manage* → *ARM template* → *Export ARM template*, or use the `Get-AzDataFactoryV2Pipeline` PowerShell cmdlet to dump each pipeline definition as JSON. You'll end up with a directory tree like: +In the Azure Data Factory portal, open *Manage* → *ARM template* → *Export ARM template*. This should create a +directory with the following structure: ```text adf-export/ @@ -23,8 +26,17 @@ adf-export/ ├── linkedService/ # one JSON file per linked service └── trigger/ # one JSON file per trigger (optional) ``` + + + +You can also use the `Get-AzDataFactoryV2Pipeline` PowerShell cmdlet to programmatically dump each pipeline definition as JSON. + + + + +## Upload the exported pipelines -Upload the directory to a Unity Catalog volume (recommended) or a local path the agent can read: +Upload the exported pipelines to a Unity Catalog volume, workspace folder, or local path: ```bash databricks fs cp -r ./adf-export dbfs:/Volumes/main/default/adf_export @@ -35,23 +47,28 @@ databricks fs cp -r ./adf-export dbfs:/Volumes/main/default/adf_export ## Run the end-to-end migration -Open a fresh conversation and prompt your agent with the path to your JSON templates and a target output directory: +Prompt your agent with the path to your JSON templates and a target output directory: -> Use flowx to migrate the ADF pipelines at `/Volumes/main/default/adf_export` into a Databricks Asset Bundle at `./flowx_output/`. +```text +flowx migrate pipelines in `/Volumes/main/default/adf_export`. Save output in `./flowx_output/`. +``` -flowx will use the `migrate` skill to chain 3 other skills. All three phases write into one shared output directory (default `./flowx_output`): +Your agent will use flowx's `migrate` skill to run an end-to-end migration consisting of several phases: -1. `discover` parses every JSON file, builds an inventory, assigns a translation strategy for each resource (deterministic, agentic, or unsupported), and emits a `metadata/profile_report.csv` complexity report (one row per pipeline). +1. `discover` parses JSON files, builds an inventory, assigns a translation strategy for each resource, and creates a complexity report. 2. `convert` converts each activity to an intermediate representation. The agent will ask for confirmation before running any LLM-based translation. -3. `package` creates a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) with job configuration files, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). +3. `package` creates a [Declarative Automation Bundle](https://docs.databricks.com/aws/en/dev-tools/bundles/resources) with job configuration, code, and setup scripts (e.g. to create Databricks Secret Scopes or Unity Catalog connections). + +To run a single step, invoke its skill directly (e.g. `@flowx-discover`) or ask your agent to run a single step. + ## Review the output -flowx writes everything into a single shared output directory (default `./flowx_output`). The generated bundle can be reviewed and modified before deployment. The layout is: +flowx writes all artifacts into a shared output folder (`./flowx_output` by default) with the following structure: ```text flowx_output/ @@ -67,21 +84,16 @@ flowx_output/ └── .work/ # transient intermediates (translation report, IR, gaps.json); pruned by prepare ``` -The bundle itself contains a top-level `databricks.yml` file with deployment targets and other variables, a `resources/` folder with job configuration, -a `src/` folder with code required to run the pipeline, and a `SETUP.md` file describing supporting resources to create. - -During translation, a transient `translation_report.json` is written under `flowx_output/.work/`. It lists every activity, its translation strategy, warnings raised during translation, and the location of any -generated artifacts. Review the translation report for any warnings, unsupported resources, or to-do items before deploying to your Databricks workspace. The `package` phase prunes `.work/` after building the bundle (pass `--keep-intermediates` to retain it). +The bundle contains: +1. `databricks.yml` file with deployment targets, global parameters, and other variables +2. `resources/` folder with job and pipeline configuration +3. `src/` folder with code required to run the pipeline +4. `SETUP.md` file that details any deployment pre-requisites -Connection strings, credentials, and other protected configuration parameters are emitted as `SecretInstruction` setup steps that require [Databricks Secrets](https://docs.databricks.com/aws/en/security/secrets/). -Run the setup scripts and populate secret values before deploying and running pipelines in your Databricks workspace. +Connection strings, credentials, and other protected configuration parameters are emitted as `SecretInstruction` steps that require [Databricks Secrets](https://docs.databricks.com/aws/en/security/secrets/). +Run the setup scripts to add any required secret values before deploying and running pipelines in your workspace. - -When running with workspace auth (e.g. Genie Code), `package` can optionally persist this run's -coverage to a Unity Catalog table — one row per pipeline stamped with a UUID `run_id`, `run_date`, -and `run_by` (`record-results`) — and install a published AI/BI coverage dashboard over that table -(`install-dashboard`). See [Configuration options](/docs/options) for details. @@ -95,9 +107,9 @@ databricks bundle validate databricks bundle deploy --target ``` - -Bundles created by flowx are standard Databricks Asset Bundles. You can target different environments, integrate with CI/CD, -or further customize the YAML before deploying. See the [Databricks Asset Bundles documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/) for more information. + +Bundles created by flowx are standard Declarative Automation Bundles. You can target different environments, integrate with CI/CD, +or further customize the YAML before deploying. See the [Declarative Automation Bundles documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/) for more information. diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index bc36385..ad294e8 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -5,22 +5,18 @@ description: What flowx is and where to start. ## Motivation -Orchestration should be treated as a first class citizen during migrations. Because orchestrators drive the execution of data pipelines, -their configuration can impact data processing results as much as the logic being orchestrated. While significant tooling exists for code -conversion and data reconciliation, migrating from legacy orchestration systems is often manual, time-consuming, and prone to risk. - flowx was created to automate migrations of data pipelines between various orchestrators. It provides a robust, tested set of capabilities to parse existing data pipeline definitions, create migration artifacts, and convert data pipeline definitions to Databricks' [Lakeflow jobs framework](https://docs.databricks.com/aws/en/jobs/). ## How flowx works -flowx is a set of agent skills and deterministic translators. Skills tell agentic tools (e.g. Databricks Genie Code, Claude Code, or any +flowx is a set of agent skills and deterministic translators. Skills tell agent tools (e.g. Databricks Genie Code, Claude Code, or any agent that supports the open [Agent Skills](https://agentskills.io/) format) how to call deterministic translators that parse, translate, and generate Databricks resources. Translation runs in three phases: -1. `discover` parses Azure Resource Manager templates (e.g. for Data Factory pipelines, datasets, linked services, and triggers) into an execution -tree, builds an inventory, and emits a per-pipeline complexity report (`metadata/profile_report.csv`). +1. `discover` parses the input pipeline configuration (e.g. ARM templates for Data Factory pipelines, datasets, linked services, and triggers) +into an execution tree, builds an inventory, and emits a per-pipeline complexity report (`metadata/profile_report.csv`). 2. `convert` processes the inventory and converts each activity into a Databricks-compatible intermediate representation. *Deterministic activities* are translated by Python handlers while *agentic activities* are handed off to an LLM-assisted translator with the right context. *Unsupported activities* are flagged as explicit gaps. @@ -33,5 +29,5 @@ All three phases write into one shared output directory (default `./flowx_output - **[Architecture](/flowx/docs/architecture)** — understand how flowx is deployed and how it translates - **[Installation](/flowx/docs/installation)** — install the flowx plugin in your agentic tool of choice. -- **[Usage Guide](/flowx/docs/guide)** — an end-to-end walkthrough from raw ADF JSON to a deployable bundle. -- **[Options](/flowx/docs/options)** — reference documenting options for customizing output when translating pipelines with flowx. +- **[Usage Guide](/flowx/docs/guide)** — run an end-to-end conversion from raw ADF JSON to a deployable bundle. +- **[Configuration](/flowx/docs/configuration)** — reference documenting options for customizing output when translating pipelines with flowx. diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index cb9f5d9..9b7ec77 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -6,69 +6,75 @@ description: Install flowx in Databricks Genie Code or a local agent harness. import { Callout } from 'fumadocs-ui/components/callout'; import { Steps, Step } from 'fumadocs-ui/components/steps'; -flowx is a set of [agent skills](https://github.com/databricks-solutions/flowx/tree/main/skills) that run from an AI coding assistant. How you install them depends on where your agent runs: +flowx is a set of [agent skills](https://github.com/databricks-solutions/flowx/tree/main/skills) that run from an AI coding assistant. Skills can be run from: -- **Databricks Genie Code** runs the phases as a hosted **MCP server** (a Databricks App). No local Python environment is involved — the app vendors flowx's code and dependencies. -- **A local agent harness (Claude Code, or any Agent Skills tool)** runs the phases from a local **Python virtual environment**, optionally exposing them over a local MCP server too. +- **Databricks Genie Code** using an MCP server hosted in Databricks Apps +- **Local agent harnesses** (e.g. Claude Code) using either a Python virtual environment or a local MCP server -Pick the matching section below and follow it end to end. +The following sections provide instructions for installing flowx using your preferred coding assistant. ## Installing flowx for Databricks Genie Code -In Genie Code the phases run as the single `flowx` tool on a Databricks App you deploy and then register as a custom MCP server. +flowx can be used directly from Databricks Genie Code. To install flowx for Genie Code, you must deploy the flowx MCP server +as a Databricks App in your workspace, then register flowx as a custom MCP server. -### Clone flowx into a shared workspace location +### Clone flowx into a shared workspace folder -Clone the repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos), under **`/Workspace/Shared`** (e.g. `/Workspace/Shared/flowx`). +First clone the repo into your workspace using a [Git folder](https://docs.databricks.com/aws/en/repos/git-operations-with-repos). - -The MCP app's service principal cannot read private `/Workspace/Users/` folders by default. Cloning into `/Workspace/Shared` keeps the repo, the deployed app source, and team access all in a location every user and the app's service principal can reach. If your workspace restricts `/Workspace/Shared`, use any other all-users location and pass it to the deployer. + +The flowx MCP app uses a service principal that cannot read private folders in `/Workspace/Users/`. Clone flowx into +`/Workspace/Shared` to allow source code access for all workspace users and service principals. If your workspace restricts +access to `/Workspace/Shared`, use an approved shared folder and configure the deployment notebook to deploy the flowx MCP +app from this shared folder. -### Copy the skills into your skills folder +### Copy the skills into your assistant folder -Genie Code picks up skills from your `.assistant/skills` folder automatically. Copy `skills/` into a user-level folder: +Genie Code references skills in your `.assistant/skills` folder automatically. These can either be installed for a single +user or for all users within a workspace. + + + +To add flowx skills for a single user, copy the skills from the flowx folder to your `.assistant` folder: ```bash -databricks workspace import-dir skills /Users//.assistant/skills +databricks workspace skills /Users//.assistant/skills ``` + -Or make flowx available to all workspace users with a workspace-level folder: + +To add flowx for all workspace users, add skills to a workspace-level folder: ```bash -databricks workspace import-dir skills /Workspace/.assistant/skills +databricks workspace skills /Workspace/.assistant/skills ``` - -Skills fire automatically when their description matches your request. To invoke one explicitly, use the `@` prefix (e.g. `@flowx-migrate translate the ADF pipelines I exported to /Volumes/main/default/adf_export`). See the [Genie Code Skills docs](https://docs.databricks.com/aws/en/genie-code/skills) for more. - - - -### Run the setup skill - -Ask your agent to *"set up the flowx environment"* (or run `@flowx-setup`). On Databricks, setup detects the environment and prepares the MCP path — it does **not** create a virtual environment, because the phases run through the deployed app rather than a local interpreter. + + -### Deploy the MCP server - -Deploy the `mcp-flowx` Databricks App from the flowx checkout. The recommended way runs entirely in the workspace, including on **serverless** compute: +### Deploy the MCP app -Open and run the **`app/deploy_app.py`** notebook. Set its `repo_root` widget to your checkout (e.g. `/Workspace/Shared/flowx`); it uses the Databricks SDK to stage a self-contained source bundle (app entrypoint plus a vendored copy of the flowx package) to `/Workspace/Shared/mcp-flowx` and create/deploy the app. The notebook prints the app URL; the MCP endpoint is **`/mcp`**. +Open and run `app/deploy_app.py`. Set the `repo_root` widget to the location of your flowx folder (e.g. `/Workspace/Shared/flowx`). +flowx uses the Databricks SDK to stage a Databricks Declarative Automation Bundle to `/Workspace/Shared/mcp-flowx` and deploy +the app. The notebook prints the app URL when deployment succeeds. The MCP endpoint is `/mcp`. - -You can instead run `bash app/deploy.sh` from a **workspace web terminal or a local machine**. The `databricks apps deploy` / `databricks sync` commands it uses require a CLI session and are **not** available from serverless notebook Python — which is why `deploy_app.py` (SDK-based) is preferred inside Genie Code. + +You can also run `bash app/deploy.sh` from a shell terminal to deploy the flowx MCP app. This requires a Databricks CLI +session and cannot be run from serverless compute. -### Grant access +### Grant access to the MCP app -- **App access:** grant **Can use** on the `mcp-flowx` app to the users or service principals that will call it (Apps UI → *Permissions*, or `databricks apps set-permissions`). -- **Data access:** grant the app's own service principal access to the catalogs, schemas, and Unity Catalog volumes the migration reads from and writes to, plus any SQL warehouse used by the reporting commands (`flowx(command="record_results")` / `flowx(command="install_dashboard")`). +Grant `CAN USE` access on the `mcp-flowx` app to any users or service principals that will use flowx. Grant the app's service +principal `READ VOLUME` access to any workspace folders or Unity Catalog volumes where your pipeline files are stored. @@ -76,125 +82,100 @@ You can instead run `bash app/deploy.sh` from a **workspace web terminal or a lo MCP servers are available in Genie Code [Agent mode](https://learn.microsoft.com/en-us/azure/databricks/genie-code/use-genie-code#modes): -1. In the Genie Code panel, click **⚙ Settings**. -2. Under **MCP Servers**, click **+ Add Server**. +1. In the Genie Code panel, click **Settings**. +2. Under **MCP Servers**, click **Add Server**. 3. Choose **Custom MCP server** and select the **`mcp-flowx`** Databricks App. 4. Click **Save**. -The single `flowx` tool is available when you use Genie Code in Agent mode. +This installs the `flowx` tool which is called when you run migrations using flowx. -A custom MCP app must be deployed in the **same workspace** and reachable at `https:///mcp`. If Genie Code cannot connect, set the app's `FLOWX_ALLOWED_ORIGINS` environment variable to your workspace URL and redeploy. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp). +A custom MCP app must be deployed in the *same workspace* and reachable at `https:///mcp`. If Genie Code cannot +connect, set the app's `FLOWX_ALLOWED_ORIGINS` environment variable to your workspace URL and redeploy. See [Connect Genie Code to MCP servers](https://learn.microsoft.com/en-us/azure/databricks/genie-code/mcp). ### Verify -Open the health endpoint `/` (returns `{"status":"ok"}`), or ask Genie Code *"what flowx MCP tools are available?"*. You should see the single `flowx` tool. You can now run `@flowx-migrate` (or the individual phase skills). +Ask Genie Code *"What flowx MCP tools are available?"*. You should see the single `flowx` tool. You can now run `@flowx-migrate` +and other flowx skills. + + +If Genie Code does not have access to the flowx MCP tool, open the your app's health endpoint at `/`. This should +return `{"status":"ok"}`). If you cannot access the health endpoint, verify the deployment steps and redeploy the app. + -## Installing flowx for a local agent harness (Claude Code) +## Installing flowx for Claude Code -Locally, flowx installs as a Claude Code plugin and runs its phases from a Python virtual environment. +flowx can also be installed with local agent harnesses (e.g. Claude Code) as an agent skills plugin. Skills can from a Python +virtual environment or locally-hosted MCP server. ### Install the plugin -flowx is distributed through its Claude Code marketplace. From a Claude Code session: +flowx is distributed through a Claude Code marketplace. It can be installed as a plugin or directly copied into your skills folder. + + ```bash /plugin marketplace add databricks-solutions/flowx /plugin install flowx@flowx +/reload-plugins ``` + -Then run `/reload-plugins` to activate it. - - + You can also copy the skill folders straight into your local skills directory: -```bash +```shell cp -R skills/{flowx-setup,flowx-discover,flowx-convert,flowx-package,flowx-migrate} ~/.claude/skills/ ``` - + + ### Run the setup skill -Run `/flowx:flowx-setup` (or ask *"set up the flowx environment"*) **once** before any phase. flowx's Python modules depend on third-party packages (`pyyaml`, `databricks-sdk`, `sqlglot`), so setup provisions an isolated virtual environment via `scripts/bootstrap.sh`. It will: +Run `/flowx:flowx-setup` to set up a local Python environment with flowx's required 3rd-party packages. The setup process will: 1. Check that `python3`, `pip`, and the `venv` module are available. 2. Create the virtual environment at `/.venv`. -3. Install `requirements.txt` into it with `pip`. -4. Write the resolved interpreter path to the marker file `/.migration-venv`, which the phase skills read. +3. Install packages listed in `requirements.txt` into the virtual environment with `pip`. +4. Write the interpreter path to a marker file (`/.migration-venv`) used when running the flowx skills. + The environment is created once and reused. No `uv` is required for plugin users. - - -If `python3`, `pip`, or the `venv` module are missing, the script prints a warning and exits **without** creating anything. Install Python, then re-run setup: - -* **macOS:** `brew install python` -* **Debian/Ubuntu:** `sudo apt-get install python3 python3-venv python3-pip` -* **Windows:** [python.org/downloads](https://www.python.org/downloads/) (enable "Add python.exe to PATH") - - - - -### (Optional) Run the phases over a local MCP server - -The phase skills call the venv CLI directly, so this step is optional. To instead drive the phases through MCP tools locally, install the MCP server stack into the venv and register the stdio server with your MCP client: - -```bash -PY="$(cat /.migration-venv)" -"$PY" -m pip install "mcp>=1.12" "uvicorn>=0.30" "starlette>=0.40" -PYTHONPATH="/src" "$PY" -m flowx.mcp -``` - -```json -{ - "mcpServers": { - "flowx": { - "command": "/.venv/bin/python", - "args": ["-m", "flowx.mcp"], - "env": { "PYTHONPATH": "/src" } - } - } -} -``` - - -If you prefer an installed package over `PYTHONPATH`, run `pip install -e ".[mcp]"` from the plugin root; then `python -m flowx.mcp` works without setting `PYTHONPATH`. ### Verify -Open Claude Code and ask *"What flowx skills do you have available?"*. You should see all five skills (`flowx-setup`, `flowx-discover`, `flowx-convert`, `flowx-package`, `flowx-migrate`). Invoke them with `/flowx:flowx-migrate`, `/flowx:flowx-discover`, etc. - -If you hit a `ModuleNotFoundError` while running a phase, the venv is missing or incomplete — re-run `/flowx:flowx-setup`. Every Python command the skills run uses the interpreter recorded in `/.migration-venv`, with `src/` on `PYTHONPATH`: +Open Claude Code and ask *"What flowx skills do you have available?"*. You should see a list of skills (e.g. `flowx-setup`, +`flowx-migrate`). You can now run `/flowx:flowx-migrate`, `/flowx:flowx-discover`, and other flows skills. -```bash -export PYTHONPATH="/src" -PY="$(cat /.migration-venv)" -"$PY" -m flowx.adapter inputs discover -``` + +If calling a skill raises a `ModuleNotFoundError`, the virtual environment is missing or incomplete. Ensure Python is installed +in your environment and that you have access to a Python package registry for installing depenedencies, then re-run `/flowx:flowx-setup`. + -### Other AI tools +### Other Agent tools -Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install the flowx skills: +Any tool that supports the [Agent Skills](https://agentskills.io/) open standard can install flowx: -1. Copy each skill folder (`skills/flowx-setup`, `skills/flowx-discover`, `skills/flowx-convert`, `skills/flowx-package`, `skills/flowx-migrate`) into the tool's configured skills directory, so the path contains `SKILL.md` directly. +1. Copy each skill folder (e.g. `skills/flowx-migrate`) into the tool's configured skills folder. Ensure the folder contains `SKILL.md` directly. 2. Restart the tool if it caches skill metadata at startup. -3. Follow the **local agent harness** setup above to provision the Python environment (`scripts/bootstrap.sh`). +3. Run the `flowx-setup` skill to create flowx's Python environment -If your tool expects a single Markdown file instead of a directory tree, concatenate the skills: +If your tool expects a single Markdown file instead of a directory tree, run the following command to concatenate the skills: ```bash cat skills/*/SKILL.md > flowx-skills.md diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index d253728..1e74e97 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -6,7 +6,7 @@ "architecture", "installation", "guide", - "options", + "configuration", "ai-tools-skills" ] } diff --git a/docs/mdx-components.tsx b/docs/mdx-components.tsx new file mode 100644 index 0000000..8ab259e --- /dev/null +++ b/docs/mdx-components.tsx @@ -0,0 +1,24 @@ +import { isValidElement } from 'react'; +import defaultComponents from 'fumadocs-ui/mdx'; +import type { MDXComponents } from 'mdx/types'; +import { CodeBlock, Pre } from 'fumadocs-ui/components/codeblock'; +import * as TabsComponents from 'fumadocs-ui/components/tabs'; + +function getCodeLanguage(children: React.ReactNode): string | undefined { + if (!isValidElement<{ className?: string }>(children)) return undefined; + const match = children.props.className?.match(/language-([a-z0-9]+)/i); + return match?.[1]; +} + +export function getMDXComponents(components?: MDXComponents): MDXComponents { + return { + ...defaultComponents, + ...TabsComponents, + pre: ({ ref: _ref, ...props }) => ( + + 
{props.children}
+ + ), + ...components, + }; +} diff --git a/docs/source.config.ts b/docs/source.config.ts index 0b6ee4d..02d05d6 100644 --- a/docs/source.config.ts +++ b/docs/source.config.ts @@ -1,7 +1,15 @@ import { defineDocs, defineConfig } from 'fumadocs-mdx/config'; +import { rehypeCodeDefaultOptions } from 'fumadocs-core/mdx-plugins'; export const docs = defineDocs({ dir: 'content/docs', }); -export default defineConfig(); +export default defineConfig({ + mdxOptions: { + rehypeCodeOptions: { + ...rehypeCodeDefaultOptions, + addLanguageClass: true, + }, + }, +}); From 22cc5f717d8f925f020efbcc61aeeb1e6df22647 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Tue, 18 Aug 2026 15:23:30 -0400 Subject: [PATCH 2/2] Fix docs hyperlinks --- docs/content/docs/guide.mdx | 4 ++-- docs/content/docs/index.mdx | 10 +++++----- docs/content/docs/installation.mdx | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/content/docs/guide.mdx b/docs/content/docs/guide.mdx index 2572aa5..ef17e20 100644 --- a/docs/content/docs/guide.mdx +++ b/docs/content/docs/guide.mdx @@ -8,8 +8,8 @@ import { Callout } from 'fumadocs-ui/components/callout'; import { Steps, Step } from 'fumadocs-ui/components/steps'; This guide walks through an end-to-end conversion of an Azure Data Factory pipeline to Lakeflow Jobs. You will install the -flowx skills in your agentic tool, hand it a directory of Azure Data Factory JSON exports, and end up with a Databricks -Asset Bundle you can deploy. +flowx skills in your agentic tool, hand it a directory of pipeline templates for conversion, and create a Declarative Automation +Bundle you can use to deploy a Databricks Lakeflow Job. diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index ad294e8..533a109 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -6,7 +6,7 @@ description: What flowx is and where to start. ## Motivation flowx was created to automate migrations of data pipelines between various orchestrators. It provides a robust, tested set of capabilities -to parse existing data pipeline definitions, create migration artifacts, and convert data pipeline definitions to Databricks' [Lakeflow jobs framework](https://docs.databricks.com/aws/en/jobs/). +to parse existing data pipeline definitions, create migration artifacts, and convert data pipeline definitions to [Databricks Lakeflow Jobs](https://docs.databricks.com/aws/en/jobs/). ## How flowx works @@ -27,7 +27,7 @@ All three phases write into one shared output directory (default `./flowx_output ## Next steps -- **[Architecture](/flowx/docs/architecture)** — understand how flowx is deployed and how it translates -- **[Installation](/flowx/docs/installation)** — install the flowx plugin in your agentic tool of choice. -- **[Usage Guide](/flowx/docs/guide)** — run an end-to-end conversion from raw ADF JSON to a deployable bundle. -- **[Configuration](/flowx/docs/configuration)** — reference documenting options for customizing output when translating pipelines with flowx. +- **[Architecture](/docs/architecture)** — understand how flowx is deployed and how it translates +- **[Installation](/docs/installation)** — install the flowx plugin in your agentic tool of choice. +- **[Usage Guide](/docs/guide)** — run an end-to-end conversion from raw ADF JSON to a deployable bundle. +- **[Configuration](/docs/configuration)** — reference documenting options for customizing output when translating pipelines with flowx. diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index 9b7ec77..2991ade 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -43,7 +43,7 @@ user or for all users within a workspace. To add flowx skills for a single user, copy the skills from the flowx folder to your `.assistant` folder: ```bash -databricks workspace skills /Users//.assistant/skills +databricks workspace import-dir skills /Users//.assistant/skills ``` @@ -51,7 +51,7 @@ databricks workspace skills /Users//.assistant/ To add flowx for all workspace users, add skills to a workspace-level folder: ```bash -databricks workspace skills /Workspace/.assistant/skills +databricks workspace import-dir skills /Workspace/.assistant/skills ``` @@ -102,7 +102,7 @@ Ask Genie Code *"What flowx MCP tools are available?"*. You should see the singl and other flowx skills. -If Genie Code does not have access to the flowx MCP tool, open the your app's health endpoint at `/`. This should +If Genie Code does not have access to the flowx MCP tool, open your app's health endpoint at `/`. This should return `{"status":"ok"}`). If you cannot access the health endpoint, verify the deployment steps and redeploy the app. @@ -110,8 +110,8 @@ return `{"status":"ok"}`). If you cannot access the health endpoint, verify the ## Installing flowx for Claude Code -flowx can also be installed with local agent harnesses (e.g. Claude Code) as an agent skills plugin. Skills can from a Python -virtual environment or locally-hosted MCP server. +flowx can also be installed with local agent harnesses (e.g. Claude Code) as an agent skills plugin. Skills can run commands +from a Python virtual environment or locally-hosted MCP server. @@ -157,11 +157,11 @@ The environment is created once and reused. No `uv` is required for plugin users ### Verify Open Claude Code and ask *"What flowx skills do you have available?"*. You should see a list of skills (e.g. `flowx-setup`, -`flowx-migrate`). You can now run `/flowx:flowx-migrate`, `/flowx:flowx-discover`, and other flows skills. +`flowx-migrate`). You can now run `/flowx:flowx-migrate`, `/flowx:flowx-discover`, and other flowx skills. If calling a skill raises a `ModuleNotFoundError`, the virtual environment is missing or incomplete. Ensure Python is installed -in your environment and that you have access to a Python package registry for installing depenedencies, then re-run `/flowx:flowx-setup`. +in your environment and that you have access to a Python package registry for installing dependencies, then re-run `/flowx:flowx-setup`.