diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac6453f2..03f560d6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,12 @@ jobs: test: name: ${{ matrix.os }}-py${{ matrix.python-version }} runs-on: ${{ matrix.os }} - timeout-minutes: 30 + # 30 was too tight once 0.3.0 grew the suite to 2372 tests: legs landed at + # 28m30s and half the matrix was CANCELLED mid-run at exactly 30m, which + # reads as a test failure but is a clock. The run is much shorter now (see + # the coverage note below); this is headroom, and a timeout never reached + # costs nothing. + timeout-minutes: 40 strategy: fail-fast: false matrix: @@ -58,15 +63,74 @@ jobs: # The single canonical Qt-free suite (testpaths=spyde/tests/migrated). # GPU tests skip themselves without CUDA/MPS. No display server required. + # + # Coverage on ONE leg only: the upload below runs solely on + # directelectron/spyde, so on every fork the other 11 legs wrote a + # coverage.xml nobody reads. Cost is +45% on a Windows dev box (measured + # 171s -> 248s over the same 25 files) but only ~15% on a hosted Linux + # runner (12.5m with vs ~10.8m without, same commit) — so this is tidiness + # and ~1.5m a leg, NOT the reason the matrix used to time out. That was + # DaskManager.shutdown() sleeping 0.5s per Session teardown; see + # test_dask_shutdown_fast.py. - name: Run tests - run: uv run pytest spyde/tests/migrated --cov=spyde --cov-report=xml + run: uv run pytest spyde/tests/migrated ${{ (matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12') && '--cov=spyde --cov-report=xml' || '' }} - name: Upload coverage to Codecov - if: ${{ github.repository_owner == 'directelectron' }} + if: ${{ github.repository_owner == 'directelectron' + && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' }} uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} + # ── Domain extras (0.3.0) ────────────────────────────────────────────────── + # The `test` job above runs with NO optional extras, which is the important + # half of the contract: SpyDE must work for a user who only does 4D-STEM, and + # the `requires_package:` gate must hide the EELS/EBSD/atoms actions rather + # than raising ImportError. This job is the other half — with exspy, + # kikuchipy and atomap installed the gated code actually executes, so a broken + # EELS/EBSD/atom path fails here instead of silently never running. + # + # One OS / one Python: the extras are pure-Python scientific packages and the + # cross-platform surface is already covered by `test`. Not part of the main + # matrix because that would quadruple install time for every leg. + test-extras: + name: extras (ubuntu, py3.12) + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.10.11" + enable-cache: true + + - name: Set up Python + run: uv python install 3.12 + + # NOT --frozen: the domain extras are not in uv.lock (they are optional + # and resolve differently per platform), so the lock cannot satisfy them. + - name: Install with all domain extras + run: uv sync --extra tests --extra all --python 3.12 + + - name: Versions + run: | + uv run python -V + # importlib.metadata, not `pkg.__version__` — atomap imports fine but + # exposes no `__version__`, which failed this job before a single test + # ran. The point of the step is "the extra installed and imports", so + # ask the installed distribution, which every package answers. + uv run python -c "import importlib.metadata as m; \ + import exspy, kikuchipy, atomap; \ + print('\n'.join(f'{p} {m.version(p)}' \ + for p in ('exspy', 'kikuchipy', 'atomap')))" + + - name: Run tests (extras present) + run: uv run pytest spyde/tests/migrated + # ── Renderer/main TypeScript typecheck ───────────────────────────────────── # Pure TS compile (tsc --noEmit) of the Electron main + renderer. No Python / # display server needed, so it's a fast standalone job that gates the PR. @@ -138,24 +202,24 @@ jobs: # Best-effort for now (continue-on-error): the e2e launches a real Electron app # + Dask LocalCluster and has not yet been validated green on a hosted runner. # Drop continue-on-error once it's confirmed stable in CI. - # Sharded ×3: the suite is deliberately serial WITHIN a run (one Electron + + # Sharded ×4: the suite is deliberately serial WITHIN a run (one Electron + # Dask cluster at a time — see playwright.config.ts), so wall-clock scales # only by splitting the spec files across independent runners. Each shard is # still fully serial, so the cluster-contention flakiness that forced # workers:1 cannot recur across shards (separate VMs). Playwright balances - # shards by FILE COUNT, not duration — ×2 measured 22.7m vs 10.0m — so ×3 - # keeps the worst shard under ~15m even while the known-failing specs pay - # retry + app-reboot tax (2026-07-11: 16 failing tests account for ~23 of - # the 33 spec-minutes; fixing them is the real speedup). + # shards by FILE COUNT, not duration — ×2 measured 22.7m vs 10.0m, and ×3 + # still ran 8.1m / 14.5m / >30m (shard 1 was CANCELLED at the timeout on + # test 76 of 115, so it produced no report at all). ×4 plus the domain + # extras below brings the worst shard back under the budget. e2e: - name: electron-e2e (ubuntu, shard ${{ matrix.shard }}/3) + name: electron-e2e (ubuntu, shard ${{ matrix.shard }}/4) runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 continue-on-error: true strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2, 3, 4] steps: - uses: actions/checkout@v4 with: @@ -175,8 +239,15 @@ jobs: version: "0.10.11" enable-cache: true + # WITH the domain extras. Without them the `requires_package:` gate hides + # every EELS/EBSD/atoms action, so the 0.3.0 specs that click those + # buttons cannot pass — 9 of them failed for that reason alone, and with + # retries:1 each one paid a second full app boot. Installing the extras + # makes them testable AND takes ~10 minutes of dead retry off shard 1. + # Not --frozen: the extras resolve per-platform, so the lock can't satisfy + # them (same reason as the `extras` job above). - name: Pre-sync the Python backend env - run: uv sync --frozen --extra tests + run: uv sync --extra tests --extra all - name: Install Electron dependencies working-directory: electron @@ -192,7 +263,7 @@ jobs: # downloads + screenshot writes) that is meant to be opt-in. - name: Run Electron e2e working-directory: electron - run: xvfb-run --auto-servernum npm run test:build -- --project=electron --shard=${{ matrix.shard }}/3 + run: xvfb-run --auto-servernum npm run test:build -- --project=electron --shard=${{ matrix.shard }}/4 - name: Upload Playwright report if: ${{ always() }} diff --git a/RELEASE_0_3_0_PLAN.md b/RELEASE_0_3_0_PLAN.md new file mode 100644 index 00000000..dce967ed --- /dev/null +++ b/RELEASE_0_3_0_PLAN.md @@ -0,0 +1,310 @@ +# SpyDE 0.3.0 — Release Plan + +Five waves, each independently shippable, each behind its own optional extra. +The tracker issue is **#48**; every wave and task below has a GitHub sub-issue. + +Prior art this plan leans on, in order of how much it saves us: +`vector_orientation_gpu.py` (batched-torch playbook), `commit.py` + +`actions/views.py` (component-map toggle), `vector_overlay.py` (interactive +point overlay), `ipf_view.py`/`orientation_map.py` (orientation display), +`actions/README.md` (the action contract — read before writing any action). + +--- + +## 0. Decisions locked before writing code + +### 0.1 `spyde/models/` is already taken + +It is the neural disk-detector package (SpotUNet + HF registry). The HyperSpy +model port therefore lives in **`spyde/fitting/`**. Do not "tidy" these +together — they are unrelated meanings of the word *model*. + +### 0.2 Package layout — one submodule per domain + +| Submodule | Wave | Extra | Contents | +|---|---|---|---| +| `spyde/fitting/` | 1 | — (core) | `ModelSpec`, torch component library, batched LM engine, seeding | +| `spyde/spectroscopy/` | 2 | `eels` | EELS/EDS composition → components, quantification | +| `spyde/ebsd/` | 3 | `ebsd` | pattern preprocessing, GPU dictionary indexing, refinement | +| `spyde/atoms/` | 4 | `atoms` | atom finding, sublattices, dumbbells, property maps | +| `spyde/data/` | 5 | — | example + synthetic datasets (owner: @CSSFrancis) | + +Interactive wiring stays in `spyde/actions/` per the existing contract +(compute in its own package, actions outside it — `find_vectors/` is the model). +No package split; these are submodules of `spyde`. + +### 0.3 Optional extras, gated at the toolbar + +```toml +[project.optional-dependencies] +eels = ["exspy>=0.3.2"] +ebsd = ["kikuchipy>=0.11.0"] +atoms = ["atomap>=0.4.1"] +all = ["spyde[eels,ebsd,atoms]"] +``` + +None of the three is installed today. Domain math comes from the real +libraries so our numbers match published results — **except** EBSD dictionary +indexing and refinement, which we implement in torch (§3) because that is the +whole point of Wave 3. + +New YAML gate key **`requires_package`**, mirroring the existing +`requires_vectors`: a toolbar entry is hidden when the extra is absent, and the +caret shows a one-line install hint instead of failing on import. This is the +only framework change Wave 0 needs. + +### 0.4 The fitting engine: batched GPU LM + seeded propagation + +Measured baseline on this box — synthetic EELS SI, 1024 spectra × 1024 +channels, power law + 2 gaussians, `multifit(optimizer="lm")`: + +| | | +|---|---| +| single `fit()` | 17.6 ms | +| `multifit` (cold, single run) | 10.83 s = 10.6 ms/spectrum = 95 spectra/s | +| `multifit` (best of 3, warm) | **9.1 ms/spectrum = 110 spectra/s** | +| extrapolated to 256×256 | **~10 min** | + +Single-threaded, one pixel at a time. That is the number to beat. + +**A trap found while measuring — do not "fix" it.** HyperSpy warns +`Numexpr is not installed, falling back to numpy, which is slower to calculate +model` on every model build. That advice is **wrong at spectrum sizes**, and +following it is a pessimisation: + +| numexpr | threads | spectra/s | +|---|---|---| +| absent | — | **110** | +| 2.14.2 | 1 | 77 | +| 2.14.2 | 4 | 75 | +| 2.14.2 | 48 | 70 | + +(best of 3 warm runs, 256 spectra × 1024 channels). numexpr's per-call setup +and thread dispatch dominate arrays of ~1k elements; it wins on large arrays, +and a single spectrum is not one. It is deliberately **not** a dependency, with +a comment in `pyproject.toml` saying so — the warning will keep inviting +someone to add it. + +The engine is the `vector_orientation_gpu.py` playbook applied to curve +fitting: pack the whole nav grid into `(P, C)` on the GPU and run one batched +Levenberg–Marquardt, no Python loop over pixels. It works because **the +parameter count `n` is tiny** (≤ ~20) even when `P` and `C` are large: the +normal equations are `(P, n, n)` batched solves, which `torch.linalg.solve` +does natively. The only big intermediate is the `(P, C, n)` Jacobian, so +**chunk over `P`** to bound it (65536 × 2048 × 12 float32 = 6.4 GB whole — +chunked it is a fixed working set). This is the same "chunk the batch +dimension" rule the orientation code already follows. + +SAMFire's insight — a neighbour's result is the best starting point — is kept +as the **seed source**, not as the scheduler: fit a strided coarse grid, then +propagate those parameters outward as initial guesses for one batched refine +over all pixels. We do not port SAMFire's per-pixel marker/strategy machinery; +that scheduler overhead is what makes it slow. + +**Acceptance gate (non-negotiable):** the GPU engine must reproduce hyperspy's +`multifit` parameters within tolerance on the same data. We are replacing a +reference implementation, so parity against it is the test — not "it converged". +CPU (scipy) fallback stays for no-GPU machines and for the parity test itself. + +--- + +## Wave 1 — Model fitting, 1D and 2D (`spyde/fitting/`) + +The foundation. Waves 2 and 4 both fit models, so they consume this engine. + +**1.1 Package skeleton + `ModelSpec`.** A serialisable description of a model +(components, parameters, bounds, free/fixed flags, signal range) that +round-trips against `hyperspy.model.BaseModel.as_dictionary()`. Storage mirrors +hyperspy — `m.store(name)` / `s.models.restore(name)` — so a saved `.hspy` +opens with its models intact in hyperspy and in SpyDE alike. Lives on the tree +as `tree.fit_models` (ownership map, `actions/README.md` §3). + +**1.2 Torch component library.** Batched, autograd-differentiable ports of +`hyperspy.components1d`: Gaussian, GaussianHF, Lorentzian, Voigt, SplitVoigt, +PowerLaw, Offset, Polynomial, Exponential, Arctan, Erf, Doniach, SkewNormal, +Logistic, HeavisideStep, ScalableFixedPattern; `components2d`: Gaussian2D, +Expression. Each takes `(x, params)` → `(P, C)` fully batched. Per-component +parity test against the hyperspy component it mirrors. + +**1.3 Batched LM engine.** `fit_batched(spec, data, ...)`: chunked over `P`, +Jacobian by autograd, bounded parameters, free/fixed masking, channel mask for +signal range, optional Poisson weighting (matters for EELS/EDS counts). +Benchmark harness in `spyde/tests/benchmark_fitting.py` reporting spectra/s +against the 95/s baseline. **Also implement variable projection**: components +linear in amplitude (most of them) are solved by batched linear least squares +given the nonlinear parameters, which shrinks `n` and improves conditioning — +this is what makes Wave 2's tabulated EELS edges tractable. + +**1.4 Seeded propagation.** Coarse strided fit → propagate to neighbours as +initial guesses → one batched refine. Reports how many pixels converged. + +**1.5 The Fit wizard.** Staged action (`fit_open/_add_component/_set_param/ +_tune/_run/_commit/_close`) per the wizard protocol. Opens a caret group; +components are added line by line; live preview fits the *current* nav +position only, so it stays interactive. `fit_run` does the whole grid on a +worker with progress. + +**1.6 Component picker with shape preview.** "Add component" shows what each +component looks like: the backend evaluates each candidate at default +parameters over the current signal axis and sends a small polyline; the +renderer draws it as a sparkline in the picker. + +**1.7 Drag-to-adjust components on the plot.** Port of the anyplotlib +interactive-fitting example: `add_point_widget` for centre+amplitude, +`add_range_widget` for width, clicking a component's line toggles its widgets, +`pointer_move` re-evaluates the sum line. This is the primary way users tune a +model — sliders are the fallback, not the main path. + +**1.8 Component area maps.** Integrated area under each component per pixel → +`commit_result_tree` with one view per component, so they toggle exactly like +εxx/εyy/εxy/ω (single click shows one, ⌘-click tiles several). This is direct +reuse of `actions/views.py`; no new display code. + +**1.9 2D models.** Same engine, image-shaped data (`Gaussian2D`, `Expression`). +Lower priority within the wave but explicitly in scope. + +--- + +## Wave 2 — EELS and EDS (`spyde/spectroscopy/`, extra `eels`) — depends on Wave 1 + +Scope distilled from the eXSpy example gallery: find EDS lines, EELS curve +fitting, background removal, Fourier-ratio deconvolution, residual plots, +quantification. + +**2.1 exspy extra + signal-type gating** (`EELS`, `EDS_TEM`, `EDS_SEM` via the +existing `signal_types` key) + microscope-parameter panel (beam energy, +collection/convergence angle) since fits are meaningless without it. + +**2.2 Composition → components.** Enter elements; auto-populate the model: +EELS gets background + an `EELSCLEdge` per ionisation edge in range; EDS gets a +gaussian per X-ray line with family ratios tied. This is the wave's headline +feature. + +**2.3 Tabulated EELS edges in the engine.** `EELSCLEdge` is a GOS lookup, not +an analytic form. Interpolate the GOS table on-device and fit edge intensities +via the linear path from 1.3 — the shapes are fixed, the amplitudes are linear. + +**2.4 Background removal + edge onset** as a RegionAction (drag the fit +window), reusing the existing region-selector machinery. + +**2.5 Interactive drag fitting** — 1.7 applied to edges/lines, which is where +it pays off most. + +**2.6 Quantification → maps.** EELS relative composition; EDS Cliff–Lorimer / +zeta-factor with absorption. Output is per-element maps → the same view-toggle +commit as 1.8. + +**2.7 Fourier-ratio deconvolution, thickness map, residual view.** + +--- + +## Wave 3 — EBSD (`spyde/ebsd/`, extra `ebsd`) + +Independent of Waves 1–2. The reuse story here is unusually good: SpyDE already +depends on **orix** and already has IPF views, orientation maps and a 3D IPF +toolbar from the 4D-STEM work, so a CrystalMap feeds straight into existing +display code. + +**3.1 kikuchipy extra, EBSD signal type, IO** (`.h5`, `.ang`, EDAX/Oxford/ +Bruker via kikuchipy's readers) + `EBSDDetector` geometry panel. + +**3.2 Pattern preprocessing**: static/dynamic background removal, neighbour +averaging, average dot-product map. These are per-pattern and embarrassingly +parallel — straight to torch. + +**3.3 GPU dictionary indexing — the reason this wave exists.** Normalized +cross-correlation between `P` experimental and `D` dictionary patterns is, once +both are zero-mean/unit-norm, a single matmul `E @ Dᵀ` → `(P, D)`, followed by +top-k. Chunk over both `P` and `D` with a running top-k so the `(P, D)` +intermediate never materialises (65k × 100k float32 = 26 GB whole). kikuchipy +does this with dask; torch should be dramatically faster. Parity against +`kikuchipy.indexing` scores is the acceptance test. + +**3.4 Orientation refinement.** Batched optimisation of `(φ1, Φ, φ2)` and +optionally the projection centre — the direct analogue of +`vector_orientation_gpu.py`. Needs differentiable master-pattern sampling +(sphere → detector bilinear interpolation) in torch; that is the real work of +this task. Watch the same numerical traps the vector-orientation work hit: +rotation-branch ambiguity and coarse-stage bias (CLAUDE.md, GPU Computing). + +**3.5 CrystalMap → existing IPF display**, phase merging +(`merge_crystal_maps`), orientation similarity map. + +**3.6 The EBSD Indexing wizard** — the whole wave made drivable, as the SAME +four-stage caret as 4D-STEM orientation mapping (Load → Library → Refine → +Run), because it is the same job on a different signal. The one thing that is +genuinely different is what the Refine stage draws: a 4D-STEM template match +is a set of SPOTS, an EBSD match is a set of BANDS, and a band centre is a +straight LINE on a flat detector — so `spyde/ebsd/bands.py` projects the +matched orientation's Kikuchi lines (and zone axes) onto the live pattern. + +Downstream of the fit nothing is new: the result is packed into a +`SpyDEOrientationMap`, so the IPF colouring, the 3-D explorer, the point +selector and the direction toggle all work unchanged, with the NCC / +orientation-similarity / ADP quality maps as chip views beside the IPF map. + +--- + +## Wave 4 — Atom position mapping (`spyde/atoms/`, extra `atoms`) — uses Wave 1's engine + +Atom refinement *is* a batched 2D gaussian fit, so 1.3 does the heavy lifting. + +**4.1 atomap extra + initial atom finding** (`get_atom_positions`), sublattice +construction, nearest neighbours, zone axes. + +**4.2 The three GUI functions, as anyplotlib overlays.** atomap's +`select_atoms_with_gui` / `add_atoms_with_gui` / +`toggle_atom_refine_position_with_gui` are matplotlib-based, so we reimplement +the *interaction* over our own overlay — which `vector_overlay.py` already does +for diffraction vectors (add/remove points, per-point state colouring). Polygon +select for phase separation; click to add/remove atoms; click to toggle an +atom's refine flag green↔red. + +**4.3 Refinement**: centre-of-mass then batched 2D gaussian via the Wave 1 +engine, honouring the per-atom refine flags. + +**4.4 Dumbbell lattices** (`get_dumbbell_vector`, `Dumbbell_Lattice`) per the +atomap dumbbell guide. + +**4.5 Property maps** — ellipticity, neighbour distance, monolayer spacing, +displacement — through the same view-toggle commit as 1.8. + +--- + +## Wave 5 — Data module (`spyde/data/`) — owner @CSSFrancis + +Datasets for 4D STEM, EELS, EDS, EBSD and in-situ. Waves 1–4 code against the +contract below and use synthetic stand-ins until real data lands, so this +never blocks them. + +**Contract** — mirror `spyde/backend/tutorial_data.py`: + +- **Bundled synthetic**, no download, fast enough for Playwright specs: + `load_test_data_` on `TestHarnessMixin`. Must be *asymmetric and + crisp* so bugs are pixel-visible (the `si_grains` / `movie` precedent). +- **Real, downloaded**: registered in the Examples menu, reached by + `load_example {name}`. +- Each entry declares `signal_type`, nav/signal shape, calibration and, where + relevant, the microscope parameters the fits need (§2.1) — a dataset that + arrives uncalibrated cannot exercise the feature it is meant to test. +- Lazy loads must use signal-spanning chunks (CLAUDE.md Live-Display §1). + +--- + +## Cross-cutting (Wave 0, do first) + +- `requires_package` gate key + install-hint caret (§0.3). +- `[project.optional-dependencies]` block + `numexpr` into core deps (§0.4). +- CI: one matrix job installing `spyde[all]` so gated code is actually + exercised; the default job proves the app still works with none of it. +- Docs: one guide per modality in `guides/`, and the data module's examples + wired into the existing tutorial menu. + +## Verification standard + +Per CLAUDE.md: a green pytest run and a clean `tsc` are **not** verification for +anything that adds windows, draws overlays or wires renderer↔backend. Every +wave's UI work ships with a Playwright spec built on `electron/tests/_harness.cjs` +and screenshots that were actually looked at. Where a wave replaces a reference +implementation (1.3 vs `multifit`, 3.3 vs `kikuchipy.indexing`), numerical +parity against that reference is the acceptance test. diff --git a/electron/playwright.config.ts b/electron/playwright.config.ts index 03e475fb..a469e635 100644 --- a/electron/playwright.config.ts +++ b/electron/playwright.config.ts @@ -3,6 +3,11 @@ import { join } from 'path' export default defineConfig({ testDir: './tests', + // Settings isolation for EVERY spec — ~30 of them call _electron.launch() + // directly rather than going through _harness.cjs, and on a fresh CI runner + // they get the first-run welcome tour whose overlay eats pointer events. + // They all spread ...process.env, so setting it there is what reaches them. + globalSetup: require.resolve('./tests/global-setup.cjs'), timeout: 120_000, expect: { timeout: 15_000 }, retries: 1, diff --git a/electron/src/main/index.ts b/electron/src/main/index.ts index 5799b9b4..f967d084 100644 --- a/electron/src/main/index.ts +++ b/electron/src/main/index.ts @@ -5,10 +5,10 @@ import { app, BrowserWindow, dialog, ipcMain, Menu, shell, nativeTheme, net, protocol, clipboard, nativeImage, } from 'electron' -import { join, basename } from 'path' +import { join, basename, resolve } from 'path' import { pathToFileURL } from 'url' import { tmpdir } from 'os' -import { existsSync, realpathSync, writeFileSync, rmSync } from 'fs' +import { existsSync, realpathSync, statSync, writeFileSync, rmSync } from 'fs' import { execFile } from 'child_process' import { startSpyDE, sendAction, sendFigureEvent, sendResize, @@ -854,6 +854,39 @@ ipcMain.on('open-external', (_, url: string) => { shell.openExternal(parsed.href) }) +// Reveal a LOCAL DIRECTORY in the OS file manager (Examples → Show Example Data +// Directory). Deliberately separate from open-external, which allowlists +// web/mail protocols precisely so it can never be talked into opening a local +// path: this one takes a filesystem path and opens it as a FOLDER only. +// `openPath` on a directory opens it in the file manager; it will not execute a +// file, and the isDirectory check means a path that somehow named an executable +// is refused rather than run. +ipcMain.on('open-path', async (_, target: string) => { + try { + const resolved = resolve(String(target ?? '')) + if (!existsSync(resolved) || !statSync(resolved).isDirectory()) { + console.warn(`[spyde] open-path rejected non-directory: ${resolved}`) + return + } + // Test seam, same shape as SPYDE_SETTINGS_DIR: handle the request and log + // it, but do not hand the path to the desktop. On a headless CI runner + // `shell.openPath` shells out to xdg-open, which has no file manager to + // reach and leaves something behind that stops the app exiting — the + // examples_menu spec's own assertion passed and then its afterAll timed + // out for 120s on app.close(). There is nothing to verify past this point + // in CI anyway ("did a file manager window appear" is not observable), so + // the spec still covers backend -> renderer -> main and stops here. + if (process.env.SPYDE_NO_SHELL_OPEN === '1') { + console.log(`[spyde] open-path (suppressed): ${resolved}`) + return + } + const err = await shell.openPath(resolved) + if (err) console.warn(`[spyde] open-path failed: ${err}`) + } catch (e) { + console.warn(`[spyde] open-path error: ${String(e)}`) + } +}) + // ── Update / GPU-status IPC ─────────────────────────────────────────────────── /** Manual "Check Now" from the update dialog. Result arrives async via the diff --git a/electron/src/preload/index.ts b/electron/src/preload/index.ts index 7bc2dd4b..fd72034c 100644 --- a/electron/src/preload/index.ts +++ b/electron/src/preload/index.ts @@ -170,6 +170,12 @@ contextBridge.exposeInMainWorld('electron', { openExternal: (url: string) => ipcRenderer.send('open-external', url), + /** Reveal a local DIRECTORY in the OS file manager (Examples → Show Example + * Data Directory). Separate from openExternal, which allowlists web/mail + * protocols so it can never open a local path; main verifies the target is + * a real directory. */ + openPath: (path: string) => ipcRenderer.send('open-path', path), + // ── Updates / GPU status ────────────────────────────────────────────────── /** Current channel, whether this build supports auto-update, last known diff --git a/electron/src/renderer/src/components/BackgroundWizard.tsx b/electron/src/renderer/src/components/BackgroundWizard.tsx new file mode 100644 index 00000000..124d0555 --- /dev/null +++ b/electron/src/renderer/src/components/BackgroundWizard.tsx @@ -0,0 +1,104 @@ +/** + * BackgroundWizard.tsx — the Remove Background caret. + * + * This existed only as a backend half. `background_action.py` has the whole + * staged set — `bg_open` / `bg_close` / `bg_set_model` / `bg_set_region` / + * `bg_apply`, a `PARAMETERS` schema registered under the `bg` key, and a + * controller that adds a green span widget to the plot and removes it in + * `remove()`. What was missing was the caret, so the action was reached + * through the plain YAML toolbar path: clicking it fired `bg_open`, which + * added the span, and nothing ever unmounted to fire `bg_close`. The span + * stayed on the plot for the life of the window. + * + * The lifecycle is not something to reimplement here — `useWizardLifecycle` + * fires the open on mount and the close on unmount, and FloatingToolbar only + * renders one caret at a time, so selecting another action unmounts this one + * and the teardown that already existed finally runs. + */ +import React from 'react' +import { WizardShell, Field, NumInput, Select, S } from './WizardShell' +import { useWizardLifecycle, useWizardEvent, useDebouncedAction } from './wizardHooks' + +interface Props { + caretPos: React.CSSProperties + windowId: number + sendAction: (action: string, payload?: Record, windowId?: number) => void + onClose: () => void +} + +const MODELS = ['PowerLaw', 'Offset', 'Polynomial', 'Exponential'] + +export function BackgroundWizard({ caretPos, windowId, sendAction, onClose }: Props) { + const [model, setModel] = React.useState('PowerLaw') + const [region, setRegion] = React.useState({ x0: 0, x1: 0 }) + const [status, setStatus] = React.useState( + 'Drag the green band over a region that is background only.') + + useWizardLifecycle({ + windowId, sendAction, + openAction: 'bg_open', + closeAction: 'bg_close', + deps: [windowId], + }) + + // The band is the PRIMARY input, so the caret follows it rather than owning + // it — dragging on the plot updates these fields, not the other way round. + useWizardEvent('spyde:bg_state', windowId, (detail) => { + const d = detail as { model?: string; x0?: number; x1?: number; status?: string } + if (d.model) setModel(d.model) + if (Number.isFinite(d.x0) && Number.isFinite(d.x1)) { + setRegion({ x0: Number(d.x0), x1: Number(d.x1) }) + } + if (d.status) setStatus(d.status) + }) + + // Typing in the fields re-places the band. Debounced: each one re-fits the + // preview, and a keystroke rate of those is work arriving faster than it is + // served — the same trap the Fit caret's navigator coalescer exists for. + const pushRegion = useDebouncedAction(sendAction, 'bg_set_region', windowId) + + const setField = (key: 'x0' | 'x1', v: number) => { + const next = { ...region, [key]: v } + setRegion(next) + pushRegion(() => next) + } + + return ( + + {/* Keep mousedown inside the caret — letting it bubble re-renders the + subwindow subtree between mousedown and mouseup, so the click never + lands. See the same note in FitWizard. */} +
e.stopPropagation()}> +
+
+ + + + + + +
+ A finer step indexes closer but costs simulation time and slows the + live preview. +
+ +
+ )} + + {tab === 'Refine' && ( +
+
+ Move the crosshair on the navigator: the matched orientation's + Kikuchi bands are drawn on the pattern. Nudge the PC until the lines + sit on the bands. +
+
+ {match + ? `NCC ${match.score.toFixed(3)} — φ1 ${match.phi1.toFixed(1)}° Φ ${match.Phi.toFixed(1)}° φ2 ${match.phi2.toFixed(1)}°` + : 'No match yet — move the crosshair.'} +
+ + { setNBands(n); tune({ nBands: n }) }} /> + + { setZoneAxes(b); tune({ zoneAxes: b }) }} /> + + { setPcx(n); tune({ pcx: n }) }} /> + + + { setPcy(n); tune({ pcy: n }) }} /> + + + { setPcz(n); tune({ pcz: n }) }} /> + +
+ )} + + {tab === 'Run' && ( +
+ + + +
+ More than one match per pattern is what the orientation-similarity + quality map needs. +
+ + {refine && ( + + + + )} + +
+ )} + + ) +} diff --git a/electron/src/renderer/src/components/FitWizard.tsx b/electron/src/renderer/src/components/FitWizard.tsx new file mode 100644 index 00000000..7d14ae3a --- /dev/null +++ b/electron/src/renderer/src/components/FitWizard.tsx @@ -0,0 +1,425 @@ +/** + * FitWizard.tsx — the Fit caret (#55, #56, #58). + * + * Two tabs, matching the Orientation-Mapping caret's shape: **Model** builds + * the model component by component, **Run** fits it and commits the maps. + * + * Layout is HORIZONTAL by preference (the same rule the toolbar popouts + * follow): a component's parameters sit side by side so the caret grows WIDER + * rather than taller. A tall caret is not just ugly here — it overhangs the + * room FloatingToolbar placed it in, and the overhang sits under the MDI area. + * + * Three deliberate choices: + * + * - **Components are added from a `+` POPUP**, not an always-open palette. The + * palette is a one-off action; leaving it permanently expanded spends the + * caret's height on something used once per component. + * - **The popup shows SHAPES, not just names** (#56). The backend samples every + * offerable component at defaults over the current signal axis and sends a + * normalised polyline, drawn as an inline SVG sparkline. "Gaussian" and + * "Lorentzian" are not distinguishable by name to someone who has not met + * them, and the point of a picker is to be able to choose. + * - **The component list is rebuilt from `fit_state`, never edited locally.** + * The backend owns the model; a caret that mutated its own copy would drift + * out of step the moment an edit was rejected and would then show a model + * that is not the one being fitted. + */ +import React from 'react' +import { WizardShell, TabRow, Field, NumInput, Select, Check, S } from './WizardShell' +import { useWizardLifecycle, useWizardEvent, CommitButton } from './wizardHooks' +import { useSpyDE } from '../kernel/SpyDEContext' + +const TABS = ['Model', 'Run'] as const +type Tab = typeof TABS[number] + +interface Props { + caretPos: React.CSSProperties + windowId: number + sendAction: (action: string, payload?: Record, windowId?: number) => void + onClose: () => void +} + +interface ParamState { name: string; value: number; free: boolean; linear: boolean } +interface CompState { + name: string; kind: string; active: boolean; parameters: ParamState[] + /** Peak height relative to the tallest component — see the header label. */ + share?: number +} +interface CatalogueItem { kind: string; description: string; preview: number[] } + +/** Inline sparkline of a component's shape — the picker's whole point. */ +function Spark({ points }: { points: number[] }) { + if (!points?.length) return null + const w = 40, h = 15 + const d = points + .map((v, i) => `${(i / (points.length - 1)) * w},${h - v * (h - 2) - 1}`) + .join(' ') + return ( + + + + ) +} + +export function FitWizard({ caretPos, windowId, sendAction, onClose }: Props) { + const { state } = useSpyDE() + const [tab, setTab] = React.useState('Model') + const [catalogue, setCatalogue] = React.useState([]) + const [components, setComponents] = React.useState([]) + const [fitted, setFitted] = React.useState(false) + const [status, setStatus] = React.useState('Add a component to begin.') + const [pickerOpen, setPickerOpen] = React.useState(false) + const [maxIter, setMaxIter] = React.useState(60) + const [seeded, setSeeded] = React.useState(true) + const [weighting, setWeighting] = React.useState<'none' | 'poisson'>('none') + const [adaptive, setAdaptive] = React.useState(false) + const [coverage, setCoverage] = React.useState({ done: 0, total: 0, here: false }) + // How many positions fit worse than their neighbours — the number the + // "Refit poor" button acts on, and the honest headline for a scan fit. + const [poor, setPoor] = React.useState(0) + // Models already stored on the signal. These are HyperSpy's own — `m.store` + // puts the components AND every position's fit into the signal's `models`, + // so they travel with the dataset rather than in a format of ours. + const [storedModels, setStoredModels] = React.useState([]) + const [modelName, setModelName] = React.useState('spyde fit') + // The elements set in Plot Control's Composition panel — what "From …" + // builds a model for. Read from the shared state rather than asked for, so + // the button appears the moment they are set. + const elements = state.composition.get(windowId)?.elements ?? [] + // Read inside the navigator listener, which is registered once — a state + // value captured there would be the value at registration forever. + const adaptiveRef = React.useRef(adaptive) + adaptiveRef.current = adaptive + + // ── navigator coalescer: one `fit_navigated` in flight, latest wins ───── + // A drag posts a pointer_move per frame and each one costs a recall, a + // preview redraw and a full model re-send. Sending one per frame queues work + // faster than it is served, and the queue drains AFTER the drag: measured 29 + // moves over 486 ms, with the model settling 1159 ms after the last of them. + // That is the pause, and the value finally arriving is the snap. + // + // Skipping intermediate sends loses nothing: `fit_navigated` reads the + // CURRENT selector position when the backend handles it, so one send after + // the previous finishes always acts on the newest position. `fit_state` + // coming back is the completion signal — self-tuning, unlike a fixed + // throttle, and there is a timeout so a lost reply cannot wedge the gate. + const inFlight = React.useRef(false) + const pending = React.useRef(false) + const wedgeTimer = React.useRef | null>(null) + + const sendNavigated = React.useCallback(() => { + if (inFlight.current) { pending.current = true; return } + inFlight.current = true + pending.current = false + if (wedgeTimer.current) clearTimeout(wedgeTimer.current) + wedgeTimer.current = setTimeout(() => { inFlight.current = false }, 2_000) + sendAction('fit_navigated', { adaptive: adaptiveRef.current }, windowId) + }, [sendAction, windowId]) + + const navDone = React.useCallback(() => { + inFlight.current = false + if (wedgeTimer.current) { clearTimeout(wedgeTimer.current); wedgeTimer.current = null } + if (pending.current) sendNavigated() + }, [sendNavigated]) + + useWizardLifecycle({ + windowId, sendAction, + openAction: 'fit_open', + closeAction: 'fit_close', + deps: [windowId], + }) + + useWizardEvent('spyde:fit_catalogue', windowId, (detail) => { + const d = detail as { components?: CatalogueItem[] } + if (d.components) setCatalogue(d.components) + }) + + useWizardEvent('spyde:fit_state', windowId, (detail) => { + const d = detail as { + components?: CompState[]; fitted?: boolean; status?: string + fitted_count?: number; nav_total?: number; position_fitted?: boolean + poor_count?: number; stored_models?: string[] + } + if (d.components) setComponents(d.components) + setFitted(Boolean(d.fitted)) + setPoor(d.poor_count ?? 0) + if (d.stored_models) setStoredModels(d.stored_models) + // Every backend edit ends in a `fit_state`, so this is the completion + // signal the navigator coalescer waits on — see sendNavigated. + navDone() + setCoverage({ + done: d.fitted_count ?? 0, + total: d.nav_total ?? 0, + here: Boolean(d.position_fitted), + }) + if (d.status) setStatus(d.status) + }) + + // ── follow the navigator ──────────────────────────────────────────────── + // The navigator's crosshair lives on a DIFFERENT window, so this listens to + // `spyde:figure_event` unfiltered and ignores anything from OUR OWN figures + // — `useWizardEvent` filters by window_id, which would drop exactly the + // events wanted here. + // + // The filter is by figId, and it has to be: `spyde:figure_event` carries + // `{figId, event}` and NO window_id at all (SpyDEContext's re-broadcast), so + // the old `detail.window_id === windowId` test compared undefined and never + // fired. Every pointer_up on this window's own plot therefore sent + // `fit_navigated` — including the ones from the caret's own drag handles. + // Before a fit that was harmless; after one the position is in the store, so + // `fit_navigated` RECALLED it and overwrote the drag the instant it landed. + // That is "once you fit a spectrum you can't move either component". + const ownFigIds = React.useRef>(new Set()) + ownFigIds.current = new Set( + (state.windows.get(windowId)?.figures ?? []).map((f) => f.figId)) + + React.useEffect(() => { + const on = (e: Event) => { + const d = (e as CustomEvent).detail as { figId?: string } + if (d.figId && ownFigIds.current.has(d.figId)) return // our own plot + sendNavigated() + } + window.addEventListener('spyde:figure_event', on) + return () => { + window.removeEventListener('spyde:figure_event', on) + if (wedgeTimer.current) clearTimeout(wedgeTimer.current) + } + }, [sendNavigated]) + + const add = (kind: string) => { + sendAction('fit_add_component', { kind }, windowId) + setPickerOpen(false) + } + + const setParam = (component: string, parameter: string, value: number) => + sendAction('fit_set_param', { component, parameter, value }, windowId) + + const toggleFree = (component: string, parameter: string, free: boolean) => + sendAction('fit_set_param', { component, parameter, free }, windowId) + + const run = () => { + setStatus('Fitting…') + setTab('Run') + sendAction('fit_run', { max_iter: maxIter, seeded, weighting }, windowId) + } + + return ( + + {/* Keep mousedown INSIDE the caret. Letting it bubble to the window + re-renders the subwindow subtree, which replaces the control being + clicked between mousedown and mouseup — the mouseup then lands on the + MDI area, React never sees a click, and the action silently does not + happen. Measured with capture-phase listeners: before this, + `mousedown -> the button's span, mouseup -> mdi-area`; after, all + three land on the span. Only the running app finds this; the handler + tests pass either way. */} +
e.stopPropagation()}> + `fit-tab-${t}`} /> + + {tab === 'Model' && ( +
+ {components.length === 0 && ( + No components yet — add one with +. + )} + + {/* One card per component; its parameters lie side by side so the + caret grows wider, not taller. */} +
+ {components.map((c) => ( +
+
+ {c.name} + {/* How TALL this component is, relative to the tallest. + The value box shows a gaussian's `A`, which is its + AREA — so a peak one sixth the height of its neighbour + but a tenth as wide reads as "888 next to 57471", i.e. + as a component suppressed to zero. It is not; this says + so. Amber below 2%, where it really has died. */} + {c.share !== undefined && ( + + {c.share >= 0.995 ? '100' + : c.share < 0.001 ? '<0.1' : (c.share * 100).toFixed(1)}% tall + + )} + +
+
+ {c.parameters.map((p) => ( +
+ {p.name} + setParam(c.name, p.name, v)} /> + toggleFree(c.name, p.name, e.target.checked)} /> +
+ ))} +
+
+ ))} +
+ + {/* ── + picker and Fit spectrum, side by side ── + "Fit spectrum" belongs HERE, not on the Run tab: building a + model is a loop — add, look, nudge, fit this one, look again — + and that loop happens entirely on this tab. Run scan is the + separate, expensive thing you do once the model is right. */} +
+ + + {/* Build the whole model from the elements in Plot Control's + Composition panel: an EELS signal gets a background and an + edge per subshell, an EDS one a background and a gaussian per + X-ray line. Everything after that is the ordinary caret — the + drag handles work on an edge exactly as on a hand-placed + gaussian. Shown only when there ARE elements; without them + the backend has nothing to build from. */} + {elements.length > 0 && ( + + )} + {/* Refit automatically as the navigator moves. Each position's + answer is remembered, so scrubbing back shows what was found + there rather than the last pixel's model. */} + + {/* Coverage, so a skipped position is visible rather than + something you discover when a map comes out patchy. */} + {coverage.total > 0 && ( + + {coverage.here ? '●' : '○'} {coverage.done}/{coverage.total} fitted + + )} + {pickerOpen && ( +
+ {catalogue.map((c) => ( + + ))} +
+ )} +
+
+ )} + + {tab === 'Run' && ( +
+ {/* Horizontal: the three knobs sit on one row. */} +
+ + setMaxIter(Math.max(5, Math.round(v)))} /> + + + setModelName(e.target.value)} + style={{ background: '#11111b', border: '1px solid #313244', + borderRadius: 4, color: '#cdd6f4', fontSize: 11, + padding: '2px 5px', width: 110 }} /> + + {storedModels.length > 0 && ( + <> +