diff --git a/dev/benchmarks/hamilton_drift_exactness_array.sh b/dev/benchmarks/hamilton_drift_exactness_array.sh new file mode 100644 index 000000000..b6c3353d8 --- /dev/null +++ b/dev/benchmarks/hamilton_drift_exactness_array.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Authoritative matched-wall A/B gate for the exact-directional EW drift scorer. +# One array task per dataset; each runs the committed harness +# dev/profiling/drift-exactness-gate-bench.R with production seeds/budgets, +# union (default) vs TS_DRIFT_EXACT, from identical per-seed local-opt starts. +# Submit chained: sbatch --dependency=afterok: this.sh +#SBATCH --job-name=ts-driftexact-gate +#SBATCH -p shared +#SBATCH -n 1 +#SBATCH --mem=4G +#SBATCH --time=4:00:00 +#SBATCH --array=0-2 +#SBATCH --output=/nobackup/%u/TreeSearch/logs/driftexact_gate_%A_%a.out +#SBATCH --error=/nobackup/%u/TreeSearch/logs/driftexact_gate_%A_%a.err + +set -e +module load r/4.5.1 +module load gcc/14.2 +export OMP_NUM_THREADS=1 +export OPENBLAS_NUM_THREADS=1 +# Dependencies + TreeTools come from the shared lib; our TreeSearch is loaded +# explicitly from LIB via the harness's lib.loc=GATE_LIB (so this shared path +# resolves deps without shadowing the branch build). +export R_LIBS_USER=/nobackup/$USER/TreeSearch/lib + +LIB=/nobackup/$USER/TreeSearch/lib-driftexact +REPO=/nobackup/$USER/TreeSearch-driftexact +OUTDIR=/nobackup/$USER/TreeSearch/drift-exactness/out +mkdir -p "$OUTDIR" +cd "$REPO" # data-raw/*.nex + the harness live here + +DATASETS=(Zhu2013 Zanol2014 Dikow2009) +DS=${DATASETS[$SLURM_ARRAY_TASK_ID]} + +# 30 seeds; nCycles sweep extended into the saturated regime to resolve the +# plateau-vs-climb crossover the 8-seed local pilot showed. +GATE_LIB="$LIB" \ +GATE_SEEDS=30 \ +GATE_NCYC=8,16,32,64,128,256 \ +GATE_DATA="$DS" \ +GATE_OUT="$OUTDIR/${DS}.csv" \ + Rscript dev/profiling/drift-exactness-gate-bench.R + +echo "DONE $DS -> $OUTDIR/${DS}.csv" diff --git a/dev/benchmarks/hamilton_drift_exactness_build.sh b/dev/benchmarks/hamilton_drift_exactness_build.sh new file mode 100644 index 000000000..9141b2d65 --- /dev/null +++ b/dev/benchmarks/hamilton_drift_exactness_build.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Build TreeSearch from the drift-exactness branch into a DEDICATED shared lib +# (does NOT clobber the cpp-search shared lib other sessions use). Chain the +# gate array on afterok of this job. +#SBATCH --job-name=ts-driftexact-build +#SBATCH -p shared +#SBATCH -n 4 +#SBATCH --mem=8G +#SBATCH --time=0:45:00 +#SBATCH --output=/nobackup/%u/TreeSearch/logs/driftexact_build_%j.out +#SBATCH --error=/nobackup/%u/TreeSearch/logs/driftexact_build_%j.err + +set -e +module load r/4.5.1 +module load gcc/14.2 + +BRANCH=claude/happy-jennings-e7a165 +LIB=/nobackup/$USER/TreeSearch/lib-driftexact +REPO=/nobackup/$USER/TreeSearch-driftexact +# Dependencies (Rcpp, TreeTools, ape, ...) live in the shared TreeSearch/lib; +# resolve them from there while installing OUR TreeSearch into the dedicated LIB +# (so we never clobber the shared lib other sessions use). +export R_LIBS_USER=/nobackup/$USER/TreeSearch/lib +mkdir -p "$LIB" /nobackup/$USER/TreeSearch/logs + +if [ ! -d "$REPO/.git" ]; then + git clone https://github.com/ms609/TreeSearch.git "$REPO" +fi +cd "$REPO" +git fetch origin "$BRANCH" +git checkout "$BRANCH" +git reset --hard "origin/$BRANCH" +echo "Git HEAD: $(git log --oneline -1)" + +rm -f src/*.o src/*.so +R CMD build --no-build-vignettes --no-manual --no-resave-data . +R CMD INSTALL --library="$LIB" TreeSearch_*.tar.gz +rc=$? +rm -f TreeSearch_*.tar.gz +echo "INSTALL exit: $rc; version: $(R_LIBS_USER=$LIB Rscript -e 'cat(as.character(packageVersion("TreeSearch")))' 2>/dev/null)" +exit $rc diff --git a/dev/benchmarks/hamilton_drift_exactness_recipe.sh b/dev/benchmarks/hamilton_drift_exactness_recipe.sh new file mode 100644 index 000000000..09c6b1c27 --- /dev/null +++ b/dev/benchmarks/hamilton_drift_exactness_recipe.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Recipe-level whole-search test (route B'): drift scorer choice in wall-limited +# thorough on TRAINING-split MorphoBank (EW-recoded). One catalogue key per task. +# Reuses lib-driftexact (no rebuild). TRAINING split only (sequestration). +#SBATCH --job-name=ts-driftexact-recipe +#SBATCH -p shared +#SBATCH -n 1 +#SBATCH --mem=8G +#SBATCH --time=12:00:00 +#SBATCH --array=0-7 +#SBATCH --output=/nobackup/%u/TreeSearch/logs/driftexact_recipe_%A_%a.out +#SBATCH --error=/nobackup/%u/TreeSearch/logs/driftexact_recipe_%A_%a.err +set -e +module load r/4.5.1; module load gcc/14.2 +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 +export R_LIBS_USER=/nobackup/$USER/TreeSearch/lib +LIB=/nobackup/$USER/TreeSearch/lib-driftexact +REPO=/nobackup/$USER/TreeSearch-driftexact +OUTDIR=/nobackup/$USER/TreeSearch/drift-exactness/out +mkdir -p "$OUTDIR"; cd "$REPO" +# Deterministic mid-size (65-135t) training-split keys, spread across sizes. +KEYS=(project3617 project589 project4624 project2124 project5201 project3807 project3602 project4614) +KEY=${KEYS[$SLURM_ARRAY_TASK_ID]} +GATE_LIB="$LIB" GATE_KEY="$KEY" GATE_CAT=/nobackup/$USER/floor/mbank_catalogue.csv \ + GATE_SEEDS=6 GATE_REPS=2,4,8 \ + GATE_OUT="$OUTDIR/recipe_${KEY}.csv" \ + Rscript dev/profiling/drift-exactness-recipe-bench.R +echo "DONE recipe $KEY" diff --git a/dev/benchmarks/hamilton_drift_exactness_replicate.sh b/dev/benchmarks/hamilton_drift_exactness_replicate.sh new file mode 100644 index 000000000..4f4ef9916 --- /dev/null +++ b/dev/benchmarks/hamilton_drift_exactness_replicate.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Route-3 gap-closer (arm A): +replicate marginal-value. Same datasets as route3 +# for direct frontier comparison. Reuses lib-driftexact (no rebuild). +#SBATCH --job-name=ts-driftexact-repl +#SBATCH -p shared +#SBATCH -n 1 +#SBATCH --mem=6G +#SBATCH --time=8:00:00 +#SBATCH --array=0-3 +#SBATCH --output=/nobackup/%u/TreeSearch/logs/driftexact_repl_%A_%a.out +#SBATCH --error=/nobackup/%u/TreeSearch/logs/driftexact_repl_%A_%a.err +set -e +module load r/4.5.1; module load gcc/14.2 +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 +export R_LIBS_USER=/nobackup/$USER/TreeSearch/lib +LIB=/nobackup/$USER/TreeSearch/lib-driftexact +REPO=/nobackup/$USER/TreeSearch-driftexact +OUTDIR=/nobackup/$USER/TreeSearch/drift-exactness/out +mkdir -p "$OUTDIR"; cd "$REPO" +DATASETS=(Zhu2013 Zanol2014 Dikow2009 mbank_X30754) +DS=${DATASETS[$SLURM_ARRAY_TASK_ID]} +SEEDS=20; if [ "$DS" = "mbank_X30754" ]; then SEEDS=10; fi +GATE_LIB="$LIB" GATE_SEEDS=$SEEDS GATE_KMAX=16 GATE_DATA="$DS" \ + GATE_OUT="$OUTDIR/repl_${DS}.csv" \ + Rscript dev/profiling/drift-exactness-replicate-bench.R +echo "DONE repl $DS" diff --git a/dev/benchmarks/hamilton_drift_exactness_route3.sh b/dev/benchmarks/hamilton_drift_exactness_route3.sh new file mode 100644 index 000000000..4172b95fa --- /dev/null +++ b/dev/benchmarks/hamilton_drift_exactness_route3.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Route-3 marginal-value experiment. One array task per dataset; runs the +# committed harness dev/profiling/drift-exactness-route3-bench.R (arms: +# ratchet / drift_exact / drift_union) from a shared per-seed local optimum, +# cycles knob, matched-wall frontier. mbank_X30754 (180t) is the large/hard +# anchor where a TBR local-opt has real room; the 74-88t sets are easy controls. +# Reuses lib-driftexact (no rebuild). +#SBATCH --job-name=ts-driftexact-route3 +#SBATCH -p shared +#SBATCH -n 1 +#SBATCH --mem=6G +#SBATCH --time=8:00:00 +#SBATCH --array=0-3 +#SBATCH --output=/nobackup/%u/TreeSearch/logs/driftexact_route3_%A_%a.out +#SBATCH --error=/nobackup/%u/TreeSearch/logs/driftexact_route3_%A_%a.err +set -e +module load r/4.5.1 +module load gcc/14.2 +export OMP_NUM_THREADS=1 +export OPENBLAS_NUM_THREADS=1 +export R_LIBS_USER=/nobackup/$USER/TreeSearch/lib +LIB=/nobackup/$USER/TreeSearch/lib-driftexact +REPO=/nobackup/$USER/TreeSearch-driftexact +OUTDIR=/nobackup/$USER/TreeSearch/drift-exactness/out +mkdir -p "$OUTDIR"; cd "$REPO" +DATASETS=(Zhu2013 Zanol2014 Dikow2009 mbank_X30754) +DS=${DATASETS[$SLURM_ARRAY_TASK_ID]} +SEEDS=20; if [ "$DS" = "mbank_X30754" ]; then SEEDS=10; fi +GATE_LIB="$LIB" GATE_SEEDS=$SEEDS GATE_CYC=1,2,4,8,16 GATE_DATA="$DS" \ + GATE_OUT="$OUTDIR/route3_${DS}.csv" \ + Rscript dev/profiling/drift-exactness-route3-bench.R +echo "DONE route3 $DS" diff --git a/dev/benchmarks/hamilton_drift_exactness_sector.sh b/dev/benchmarks/hamilton_drift_exactness_sector.sh new file mode 100644 index 000000000..e8b137324 --- /dev/null +++ b/dev/benchmarks/hamilton_drift_exactness_sector.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Sector-godrift isolation gate (route B). One array task per dataset; runs the +# committed harness dev/profiling/drift-exactness-sector-bench.R (3 arms: +# none/union/exact) at production seeds/replicates. Reuses lib-driftexact (code +# unchanged since f7f17e16 -- docs-only commits since), so NO rebuild: just +# `git pull` the repo for the new harness, then submit this. +#SBATCH --job-name=ts-driftexact-sector +#SBATCH -p shared +#SBATCH -n 1 +#SBATCH --mem=6G +#SBATCH --time=8:00:00 +#SBATCH --array=0-3 +#SBATCH --output=/nobackup/%u/TreeSearch/logs/driftexact_sector_%A_%a.out +#SBATCH --error=/nobackup/%u/TreeSearch/logs/driftexact_sector_%A_%a.err +set -e +module load r/4.5.1 +module load gcc/14.2 +export OMP_NUM_THREADS=1 +export OPENBLAS_NUM_THREADS=1 +export R_LIBS_USER=/nobackup/$USER/TreeSearch/lib +LIB=/nobackup/$USER/TreeSearch/lib-driftexact +REPO=/nobackup/$USER/TreeSearch-driftexact +OUTDIR=/nobackup/$USER/TreeSearch/drift-exactness/out +mkdir -p "$OUTDIR" +cd "$REPO" +DATASETS=(Zhu2013 Zanol2014 Dikow2009 mbank_X30754) +DS=${DATASETS[$SLURM_ARRAY_TASK_ID]} +# Fewer seeds for the 180-tip mbank (each thorough search is far slower). +SEEDS=15; if [ "$DS" = "mbank_X30754" ]; then SEEDS=8; fi +GATE_LIB="$LIB" GATE_SEEDS=$SEEDS GATE_REPS=1,2,4 GATE_DATA="$DS" \ + GATE_OUT="$OUTDIR/sector_${DS}.csv" \ + Rscript dev/profiling/drift-exactness-sector-bench.R +echo "DONE sector $DS" diff --git a/dev/profiling/drift-exactness-census.R b/dev/profiling/drift-exactness-census.R new file mode 100644 index 000000000..6978b95b6 --- /dev/null +++ b/dev/profiling/drift-exactness-census.R @@ -0,0 +1,64 @@ +#!/usr/bin/env Rscript +# Drift EW decision-flip census: quantify how far the deployed union-of-finals +# drift scorer diverges from the exact directional path, on the three gate +# datasets recoded to pure Fitch (EW). Prints the DRIFT-SCANCHK line for the +# union (default) and exact paths. Local, bounded -- severity probe only. +suppressMessages({ + library(TreeSearch, lib.loc = ".agent-hj") + library(TreeTools) +}) + +fitchPhy <- function(p) { + m <- PhyDatToMatrix(p, ambigNA = FALSE) + m[m == "-"] <- "?" # inapplicable -> missing => standard Fitch + MatrixToPhyDat(m) +} + +makeDs <- function(dataset) { + at <- attributes(dataset) + list(contrast = at$contrast, + tip_data = matrix(unlist(dataset, use.names = FALSE), + nrow = length(dataset), byrow = TRUE), + weight = at$weight, + levels = at$levels) +} + +datasets <- c("Zhu2013", "Zanol2014", "Dikow2009") + +for (nm in datasets) { + phy <- fitchPhy(ReadAsPhyDat(file.path("data-raw", paste0(nm, ".nex")))) + ds <- makeDs(phy) + nTip <- length(phy) + hasGap <- "-" %in% ds$levels + cat(sprintf("\n=== %s : %d tips, %d chars(weighted), levels={%s} hasGap=%s ===\n", + nm, nTip, sum(ds$weight), paste(ds$levels, collapse = ""), hasGap)) + + # Production-representative start: random tree -> TBR to a local optimum, then + # census drift from there (drift runs AFTER a hill-climb in real searches). + set.seed(1L) + start <- RandomTree(nTip, root = TRUE) + # Align edge tip index i with tip_data row i (= names(phy)[i]). + start$tip.label <- names(phy) + tbr <- TreeSearch:::ts_tbr_search(start$edge, ds$contrast, ds$tip_data, + ds$weight, ds$levels, maxHits = 5L) + startTree <- list(edge = tbr$edge) + cat(sprintf(" local-opt score after TBR = %.0f\n", tbr$score)) + + runDrift <- function(exact) { + if (exact) Sys.setenv(TS_DRIFT_EXACT = "1") else Sys.unsetenv("TS_DRIFT_EXACT") + Sys.setenv(TS_DRIFT_SCANCHK = "1") + set.seed(42L) + r <- TreeSearch:::ts_drift_search(startTree$edge, ds$contrast, ds$tip_data, + ds$weight, ds$levels, + nCycles = 12L, afdLimit = 3L, + rfdLimit = 0.1, maxHits = 1L) + Sys.unsetenv("TS_DRIFT_SCANCHK"); Sys.unsetenv("TS_DRIFT_EXACT") + cat(sprintf(" [%s] final drift score = %.0f (drift_moves=%d tbr_moves=%d)\n", + if (exact) "EXACT" else "UNION", r$score, + r$total_drift_moves, r$total_tbr_moves)) + invisible(r) + } + runDrift(FALSE) # union (deployed default) -- severity + runDrift(TRUE) # exact -- wiring validation (applied_mismatch must be 0) +} +cat("\nDONE\n") diff --git a/dev/profiling/drift-exactness-frontier-combined.R b/dev/profiling/drift-exactness-frontier-combined.R new file mode 100644 index 000000000..0213c81d0 --- /dev/null +++ b/dev/profiling/drift-exactness-frontier-combined.R @@ -0,0 +1,27 @@ +# Combined frontier: original nc{8..256} + extension nc{512,1024}, matched wall. +dir <- "dev/profiling" +read1 <- function(f) read.csv(file.path(dir, f)) +scoreAtWall <- function(d, W) { + sd <- unique(d$seed) + v <- sapply(sd, function(s){x<-d[d$seed==s & d$wall_s<=W,]; if(nrow(x)==0) NA else min(x$score)}) + mean(v, na.rm=TRUE) +} +for (nm in c("Zhu2013","Zanol2014")) { + df <- rbind(read1(sprintf("drift-exactness-gate-hamilton-%s.csv", nm)), + read1(sprintf("drift-exactness-gate-hamilton-%s-ext.csv", nm))) + cat(sprintf("\n== %s : mean score / mean wall by nCycles x scorer ==\n", nm)) + ag <- aggregate(cbind(score,wall_s)~nCycles+scorer, df, mean) + ag <- ag[order(ag$scorer, ag$nCycles),]; ag$score<-round(ag$score,2); ag$wall_s<-round(ag$wall_s,2) + print(ag, row.names=FALSE) + cat(sprintf("\n-- %s matched-WALL frontier (mean over 30 seeds) --\n", nm)) + cat(sprintf(" %6s %9s %9s %s\n","wall","union","exact","winner(margin)")) + for (W in c(0.8,1.6,3.2,6.4,9,12,16,20)) { + u<-scoreAtWall(df[df$scorer=="union",],W); e<-scoreAtWall(df[df$scorer=="exact",],W) + win <- if(any(is.na(c(u,e)))) "-" else if(abs(u-e)<1e-9) "tie" else if(e in wall); mean and +# best over the 30 seeds. Reports union vs exact and the crossover per dataset. +dir <- "dev/profiling" +files <- list.files(dir, pattern = "^drift-exactness-gate-hamilton-.*\\.csv$", + full.names = TRUE) +df <- do.call(rbind, lapply(files, read.csv)) +cat(sprintf("rows=%d datasets=%s seeds=%d nCycles=%s\n\n", + nrow(df), paste(unique(df$dataset), collapse=","), + length(unique(df$seed)), paste(sort(unique(df$nCycles)), collapse=","))) + +# Raw mean score + mean wall by dataset x nCycles x scorer (matched-cycles view) +cat("== mean score / mean wall(s) by dataset x nCycles x scorer ==\n") +ag <- aggregate(cbind(score, wall_s) ~ dataset + nCycles + scorer, df, mean) +ag <- ag[order(ag$dataset, ag$nCycles, ag$scorer), ] +ag$score <- round(ag$score, 2); ag$wall_s <- round(ag$wall_s, 3) +print(ag, row.names = FALSE) + +# Matched-WALL frontier +scoreAtWall <- function(d, W) { + seeds <- unique(d$seed) + v <- sapply(seeds, function(s){ + x <- d[d$seed==s & d$wall_s<=W, ]; if (nrow(x)==0) NA else min(x$score) + }) + c(mean=mean(v, na.rm=TRUE), best=min(v, na.rm=TRUE), cov=mean(!is.na(v))) +} +Ws <- c(0.1,0.2,0.4,0.8,1.6,3.2,6.4,12.8) +for (nm in unique(df$dataset)) { + cat(sprintf("\n== %s : matched-WALL frontier (mean score over 30 seeds; * = coverage<1) ==\n", nm)) + cat(sprintf(" %6s %10s %10s %s\n","wall","union","exact","winner(margin)")) + for (W in Ws) { + u <- scoreAtWall(df[df$dataset==nm & df$scorer=="union",], W) + e <- scoreAtWall(df[df$dataset==nm & df$scorer=="exact",], W) + mk <- function(z) if (z["cov"]<1) "*" else " " + win <- if (any(is.na(c(u["mean"],e["mean"])))) "-" else + if (abs(u["mean"]-e["mean"])<1e-9) "tie" else + if (e["mean"]0` +confirms the EW path ran (a stray inapplicable would route to NA and score 0 +candidates). Start = random tree → `ts_tbr_search` to a local optimum → 12 +drift cycles, `afdLimit=3`. + +## Results (`drift-exactness-census.out`) + +Per-candidate error `= union − exact`; `select_flip` = union's chosen move is +exact-suboptimal; `env_violation` = union admits a move whose TRUE cost exceeds +`afdLimit` (drift leaves its envelope); `applied under` = applied moves that are +truly WORSE than the scan claimed (optimizer's-curse signature). + +| dataset (EW) | path | over % (max) | under % (max) | select_flip | env_violation | applied under/checked | applied maxdiff | +|---|---|---|---|---|---|---|---| +| Zhu2013 (75t) | union | 45.2% (69) | **49.0% (100)** | 80.4% | **62.3%** | 127/138 | 83 | +| Zhu2013 | exact | 43.0% (39) | 49.7% (86) | 78.4% | 68.3% | **0/360** | **0** | +| Zanol2014 (74t) | union | 49.9% (30) | 44.2% (52) | 78.6% | 52.1% | 128/135 | 42 | +| Zanol2014 | exact | 47.9% (27) | 45.5% (54) | 72.9% | 54.9% | **0/286** | **0** | +| Dikow2009 (88t) | union | 66.2% (15) | 8.6% (16) | 31.6% | 7.6% | 129/170 | 7 | +| Dikow2009 | exact | 65.0% (16) | 10.6% (15) | 30.7% | 11.7% | **0/344** | **0** | + +- **Exact path validates the wiring**: `applied_mismatch == 0`, `maxdiff == 0` + on all three (the exact scan's predicted `best_candidate` equals the + post-apply `full_rescore` for every applied move — SPR and reroot). +- **Union is two-sided**: e.g. Zhu2013 under-counts 49% of candidates, by up to + **100 steps**. (The over/under % on the *exact* path measure what union WOULD + have done along that trajectory — same conclusion, different walk.) +- **Optimizer's curse is explicit**: of the union path's mis-scored *applied* + moves, 127/128 (Zhu), 128/132 (Zanol), 129/170 (Dikow) are UNDER-counts — the + selected moves are overwhelmingly the ones the scan was optimistic about. +- **Envelope violations**: on Zhu2013/Zanol2014 the union path admits a move + whose true cost exceeds `afdLimit` on the *majority* of clips (62%, 52%). + +Single-seed final drift scores (NOT a gate; fixed cycles, not matched wall): +Zhu2013 625≡625, Zanol2014 union **1264** vs exact 1266, Dikow2009 union 1609 vs +exact **1606**. Mixed → consistent with "diversifier: expect wash". Note the +interleaved-TBR workload differs hugely (union `tbr_moves` 331/251/45 vs exact +23/26/26): union drift lands in worse places that the exact TBR phase then has +to climb out of. + +## Why the exactness gate said "strict over-count" but drift shows under-counts + +The gate (`exactness-gate.md`, P2) recomputed **fresh** finals on the clipped +tree, isolating the *simplified-final formula* error (a bounded over-count ≤5). +Deployed drift reads `tree.final_` maintained by `fitch_incremental_uppass`, +which refreshes only the clip-path nodes; off-path finals carry stale content, +so the union scan errs in both directions by much larger margins. **The precise +per-candidate error source (stale finals vs. the union formula on large +multistate trees) is not separately verified here and is not needed:** the +harm follows from selection bias regardless of source, and the exact scan +removes it by construction (`applied_mismatch == 0`). + +## Fix scope + +- `ts_drift.cpp` — pure-EW SPR (`:505`→`score_ew`) and reroot (`:563`→`score_ew`) + loops, `edge_set_buf`/`up`/`pre` hoisted to function scope, per-clip + `compute_insertion_edge_sets`. EW-only; NA/IW unchanged. +- `ts_search.cpp` `spr_search` — pure-EW candidate scan (`:365`), same shape, + behind `TS_SPR_EXACT`. +- Verified diagnostic-only (unchanged): `ts_rcpp.cpp:794` (returns a + `match` field), `ts_rcpp.cpp:2660` (timing harness, return discarded). + `ts_temper.cpp:308` is a legitimate approximate ranker with exact re-score. + +## Local directional pilot (`drift-exactness-gate-bench.R`, `-pilot.out`) + +8 seeds x nCycles {8,24,64}, same per-seed local-opt start, identical RNG stream +per path (`set.seed` before each drift run) so the ONLY difference is the scorer. +`nCycles` is the replicate-like policy knob; wall is the eval metric +(`policy-in-replicates-not-seconds`). Mean score / mean wall(s): + +| dataset | nCyc | exact score | union score | exact wall | union wall | +|---|---|---|---|---|---| +| Zhu2013 | 8 | **625.50** | 626.75 | **0.079** | 0.220 | +| Zhu2013 | 24 | **625.13** | 625.25 | **0.224** | 0.668 | +| Zhu2013 | 64 | 625.13 | **624.38** | **0.555** | 1.698 | +| Zanol2014 | 8 | **1263.13** | 1263.88 | **0.099** | 0.273 | +| Zanol2014 | 24 | **1262.38** | 1262.63 | **0.325** | 0.784 | +| Zanol2014 | 64 | 1262.25 | **1261.88** | **0.823** | 2.244 | +| Dikow2009 | 8 | **1607.38** | 1608.38 | **0.201** | 0.210 | +| Dikow2009 | 24 | **1606.75** | 1607.50 | **0.599** | 0.663 | +| Dikow2009 | 64 | **1606.25** | 1606.63 | 1.615 | 1.668 | + +### Matched-WALL frontier (score achievable by wall ≤ W, mean over 8 seeds) + +The fair comparison is score-at-matched-**wall**, not matched-cycles (exact is +faster per cycle, so matched-cycles understates it at low wall and overstates it +at high wall). Frontier from the same CSV: + +| dataset | W=0.3s | W=0.6s | W=1.0s | W=1.6s | W=2.5s | +|---|---|---|---|---|---| +| Zhu2013 union | 626.71 | 626.75 | 625.25 | **624.62** | **624.38** | +| Zhu2013 exact | **625.12** | **625.12** | **625.12** | 625.12 | 625.12 | +| Zanol2014 union | 1263.71 | 1263.88 | 1262.62 | 1262.62 | **1261.88** | +| Zanol2014 exact | **1263.12** | **1262.38** | **1262.25** | **1262.25** | 1262.25 | +| Dikow2009 union | 1608.38 | 1608.25 | 1607.50 | 1606.88 | 1606.62 | +| Dikow2009 exact | **1607.43** | **1607.25** | **1606.75** | **1606.50** | **1606.25** | + +**There is a frontier CROSSOVER, not a uniform win** (advisor-flagged; my initial +matched-cycles read missed it): + +1. **Fast/wall-race regime — exact wins on all three.** Exact reaches a + given score at less wall (it amortizes `compute_insertion_edge_sets`/clip + over cheap cached scans; the ~2.9× whole-`drift_search` wall gap on Zhu/Zanol + also reflects union drifting to worse places so the interleaved TBR phases do + more cleanup — 331 vs 23 `tbr_moves` — i.e. faster *and* shallower are one + coin, not two wins). This is the `auto` objective: **no time-to-optimum + regression.** +2. **Saturated/thorough regime — union wins 2/3.** Exact *plateaus* (Zhu 625.12 + from nc≥24; Zanol 1262.25), while union keeps *climbing* (Zhu → 624.38; Zanol + → 1261.88) at ~3× the wall. This is structural across the whole sweep, not + nc-64 noise: union's productive wrongness admits falsely-cheap moves → + over-diversifies → escapes to a deeper optimum the interleaved exact-TBR then + captures. Exact's cleaner hill-climb converges faster but shallower. Dikow2009 + shows no crossover — exact dominates every budget. + +This is precisely the `auto` vs `thorough` split (`auto-vs-thorough-objective`): +exact is the better `auto` scorer; union has a small saturated-reach edge on 2/3 +for `thorough`. + +### Authoritative Hamilton gate — 30 seeds, nCycles {8..256} (CONFIRMS the crossover) + +Run on Hamilton (build job 17870086 → array 17870087, one task/dataset; +`dev/benchmarks/hamilton_drift_exactness_{build,array}.sh`; raw CSVs +`drift-exactness-gate-hamilton-*.csv`; reanalyse with +`drift-exactness-frontier.R`). Matched-**wall** frontier, mean score over 30 +seeds: + +| dataset | W=0.2s | W=0.4s | W=0.8s | W=1.6s | W=3.2s | W=6.4s | winner path | +|---|---|---|---|---|---|---|---| +| Zhu2013 union | 626.6 | 626.1 | 625.3 | **624.6** | **624.4** | **624.2** | exact<0.8s, union≥1.6s | +| Zhu2013 exact | **626.2** | **625.6** | **625.2** | 624.9 | 624.8 | 624.8 | (plateaus 624.8) | +| Zanol2014 union | 1264.0 | 1263.5 | 1262.7 | 1262.0 | **1261.7** | **1261.5** | exact≤1.6s, union≥3.2s | +| Zanol2014 exact | **1263.3** | **1262.6** | **1262.3** | **1261.9** | 1261.9 | 1261.9 | (plateaus 1261.9) | +| Dikow2009 union | 1608.1 | 1608.1 | 1607.6 | 1607.2 | 1607.1 | 1606.8 | **exact all budgets** | +| Dikow2009 exact | **1608.0** | **1607.8** | **1607.2** | **1606.9** | **1606.7** | **1606.6** | (no crossover) | + +- **Exact wins the `auto`/wall-race regime on all three** — and by a large wall + margin (at nc=256: Zanol exact 3.08s vs union 8.28s; Zhu 2.03s vs 6.27s; + ~2.7–3× faster per matched cycle). No time-to-optimum regression anywhere. + This is solid and unaffected by the caveat below. +- **The nc≤256 saturated tail was censoring-confounded — extended to nc1024 to + settle it.** Both scorers stop at nc256, but exact runs ~3× faster, so its + nc256 point (~2s) was being compared against union's ~6–8s point. So exact+union + were re-run at nc {512,1024} on Zhu2013/Zanol2014 (30 seeds; + `drift-exactness-gate-hamilton-*-ext.csv`; combined reanalysis + `drift-exactness-frontier-combined.R`). Result: exact does **not** plateau (Zhu + nc256→512→1024 = 624.80→624.73→624.40; Zanol 1261.87→1261.73→1261.50) — it keeps + descending, just *slower* than union. At genuinely matched wall (both with real + points in 3–12s) **union stays ahead throughout the tail**: Zhu W=6.4s union + 624.17 vs exact 624.73, W=12s 624.17 vs 624.40, union floor 624.0 vs exact ~624.4; + Zanol union ahead by a narrower ~0.13. So the crossover is **real, not an + artifact** — union genuinely wins the thorough regime on both (clearly on Zhu). +- **best-of-30 is equal on all three** (Zhu 624, Zanol 1261, Dikow 1606): both + scorers *can* reach the same optimum — a reliability/wall difference, not + reachability. + +### spr_search — PROMOTED to exact default (large win, no crossover) + +`spr_search` is a pure hill-climb with no compensating exact-TBR phase, so the +union over-count (inflating `best_candidate` past the `dominated` gate, +`ts_search.cpp:379`) makes it converge **catastrophically short** with nothing +to recover. Multistart matched-wall check (25 random starts, run to convergence): + +| dataset | union best-of-25 | exact best-of-25 | union wall/start | exact wall/start | +|---|---|---|---|---| +| Zhu2013 | 650 | **625** | 0.29s | **0.22s** | +| Zanol2014 | 1292 | **1265** | 0.41s | **0.32s** | + +Union's shortfall (25–70 steps) is a *systematic* premature-convergence floor, +not variance — union's best-of-many can't approach exact's single-start reach. +And exact is **faster per start** (cheaper cached scan + cleaner convergence). +Best-of-starts favours exact at every matched-wall budget; **no crossover** +(unlike drift — spr_search has no exact-TBR phase to launder union's +over-diversification into reach). Wiring validated (returned score == +independent `ts_fitch_score` on every run). **Landed exact as the DEFAULT** +(`ts_search.cpp`); `TS_SPR_UNION` restores the old scorer (kill switch). Reach: +the `sprFirst` warmup is OFF in all presets, so this affects the standalone +`ts_spr_search` today — but it also makes an SPR warmup *viable*, so +`sprFirst=TRUE` in a fast preset is now worth its own gate. + +## Gate — CLOSED. Verdict: LAND OPT-IN, do not flip the drift default + +Settled by the 30-seed Hamilton gate + the nc1024 censoring check: +- **Exact wins the `auto`/wall-race regime decisively on all three** (no + time-to-optimum regression, ~2.7–3× faster) — justifies landing `TS_DRIFT_EXACT` + and, as a follow-up, having the `auto`/fast presets enable it. +- **Union wins the `thorough`/saturated regime on 2/3** (Zhu2013 clearly, + Zanol2014 narrowly; Dikow no crossover) at genuinely matched wall — confirmed + real, not a censoring artifact. +Per the rule (`auto-vs-thorough-objective`: thorough disqualifier = saturated-reach +regression), a scorer that regresses saturated reach on 2/3 must not be the global +default. `TS_DRIFT_EXACT` lands **opt-in**; union's productive-wrongness +diversification stays the default for `thorough`. + +`spr_search` is the opposite case — a large unambiguous reach win, no crossover +(the union over-count trips the `dominated` gate and prematurely converges, with +no exact-TBR phase to recover). **Promoted to exact default** (kill switch +`TS_SPR_UNION`); see the spr_search section for the multistart evidence. + +The correctness/severity finding (no score leak; two-sided error + optimizer's +curse) is independent of and unaffected by the gate outcome. diff --git a/dev/profiling/drift-exactness-recipe-bench.R b/dev/profiling/drift-exactness-recipe-bench.R new file mode 100644 index 000000000..33a37cdcf --- /dev/null +++ b/dev/profiling/drift-exactness-recipe-bench.R @@ -0,0 +1,52 @@ +#!/usr/bin/env Rscript +# Recipe-level whole-search test (route B'): does the drift SCORER choice change +# a wall-limited thorough search on TRAINING-split MorphoBank data? One catalogue +# key per invocation (GATE_KEY), EW-recoded (-> Fitch). Three arms, sector drift +# OFF (sectorGoDrift=0) to isolate the MAIN-tree drift phase: +# none : driftCycles=0 (drift off; budget goes to replicates/other phases) +# union : driftCycles=3, union-of-finals scorer (current default) +# exact : driftCycles=3, TS_DRIFT_EXACT exact directional scorer +# maxReplicates is the wall knob; same seed per arm => matched RNG. TRAINING +# split only (validation-set-sequestered). +suppressMessages({ + library(TreeSearch, lib.loc = Sys.getenv("GATE_LIB", ".agent-hj")) + library(TreeTools) +}) +key <- Sys.getenv("GATE_KEY") +catpath <- Sys.getenv("GATE_CAT", "/nobackup/pjjg18/floor/mbank_catalogue.csv") +mdir <- Sys.getenv("GATE_MATRICES", "") # resolved below if empty +seeds <- seq_len(as.integer(Sys.getenv("GATE_SEEDS", "6"))) +reps <- as.integer(strsplit(Sys.getenv("GATE_REPS", "2,4,8"), ",")[[1]]) +outfile <- Sys.getenv("GATE_OUT", sprintf("dev/profiling/recipe_%s.csv", key)) +driftN <- as.integer(Sys.getenv("GATE_DRIFT", "3")) + +cat <- read.csv(catpath, stringsAsFactors = FALSE) +row <- cat[cat$key == key, ] +if (nrow(row) != 1) stop("key not found / ambiguous: ", key) +if (row$split != "training") stop("REFUSING non-training split for tuning: ", key, " is ", row$split) +if (mdir == "") for (d in c("../neotrans/inst/matrices","../neotrans/matrices")) + if (dir.exists(d)) { mdir <- normalizePath(d); break } +nex <- file.path(mdir, row$filename) +if (!file.exists(nex)) stop("matrix not found: ", nex) + +fitchPhy <- function(p){m<-PhyDatToMatrix(p,ambigNA=FALSE); m[m=="-"]<-"?"; MatrixToPhyDat(m)} +phy <- fitchPhy(ReadAsPhyDat(nex)); nt <- length(phy) +cat(sprintf("KEY %s : %d tips (recoded EW), catalogue ntax=%d pct_inapp=%.1f\n", + key, nt, row$ntax, row$pct_inapp)) + +rows <- list() +for (s in seeds) for (R in reps) for (arm in c("none","union","exact")) { + dc <- if (arm == "none") 0L else driftN + if (arm == "exact") Sys.setenv(TS_DRIFT_EXACT="1") else Sys.unsetenv("TS_DRIFT_EXACT") + set.seed(s); t0 <- Sys.time() + res <- MaximizeParsimony(phy, strategy="thorough", driftCycles=dc, sectorGoDrift=0L, + maxReplicates=R, nThreads=1L, verbosity=0L) + wall <- as.double(difftime(Sys.time(), t0, units="secs")) + Sys.unsetenv("TS_DRIFT_EXACT") + sc <- as.double(attr(res, "score")) + rows[[length(rows)+1L]] <- data.frame(key=key, nTip=nt, seed=s, reps=R, arm=arm, + wall_s=round(wall,3), score=sc) + cat(sprintf("%-16s seed=%d reps=%d %-5s score=%.0f wall=%.2fs\n", key, s, R, arm, sc, wall)) +} +write.csv(do.call(rbind, rows), outfile, row.names=FALSE) +cat(sprintf("Wrote %s\n", outfile)) diff --git a/dev/profiling/drift-exactness-replicate-bench.R b/dev/profiling/drift-exactness-replicate-bench.R new file mode 100644 index 000000000..4e7a2f0bd --- /dev/null +++ b/dev/profiling/drift-exactness-replicate-bench.R @@ -0,0 +1,54 @@ +#!/usr/bin/env Rscript +# Route-3 gap-closer: the +replicate arm. Marginal-value asks "is a wall-second +# better spent on exact-drift than on the incumbent lever?" -- and this project's +# real incumbent is MORE REPLICATES (policy-in-replicates-not-seconds: +# maxReplicates anytime-dominates), not ratchet. From the SAME per-seed local +# optimum L (identical construction to drift-exactness-route3-bench.R), spend the +# budget on k additional independent RAS+TBR restarts, tracking best-of(L, r1..rk) +# vs cumulative wall. Merge with route3_*.csv and compare frontiers: if +# drift_exact still beats `replicate` at matched wall -> drift earns recipe space. +suppressMessages({ + library(TreeSearch, lib.loc = Sys.getenv("GATE_LIB", ".agent-hj")) + library(TreeTools) +}) +seeds <- seq_len(as.integer(Sys.getenv("GATE_SEEDS", "20"))) +kmax <- as.integer(Sys.getenv("GATE_KMAX", "16")) # max extra restarts +dnames <- strsplit(Sys.getenv("GATE_DATA", "Zhu2013,Zanol2014,Dikow2009"), ",")[[1]] +outfile <- Sys.getenv("GATE_OUT", "dev/profiling/drift-exactness-replicate-bench.csv") + +fitchPhy <- function(p){m<-PhyDatToMatrix(p,ambigNA=FALSE); m[m=="-"]<-"?"; MatrixToPhyDat(m)} +loadPhy <- function(nm){ + for (p in c(file.path("data-raw",paste0(nm,".nex")), file.path("dev/benchmarks",paste0(nm,".nex")))) + if (file.exists(p)) return(fitchPhy(ReadAsPhyDat(p))) + stop("not found: ", nm) +} +tbrTo <- function(edge, ds) TreeSearch:::ts_tbr_search(edge, ds$contrast, ds$tip_data, ds$weight, ds$levels, maxHits=1L) + +rows <- list() +for (nm in dnames) { + phy <- loadPhy(nm); nt <- length(phy); at <- attributes(phy) + ds <- list(contrast=at$contrast, tip_data=matrix(unlist(phy,use.names=FALSE),nrow=nt,byrow=TRUE), + weight=at$weight, levels=at$levels) + for (s in seeds) { + # L: identical to the route-3 harness (same seed => same shared start). + set.seed(s); st <- RandomTree(nt, root=TRUE); st$tip.label <- names(phy) + t0 <- Sys.time(); Lr <- tbrTo(st$edge, ds); L_wall <- as.double(difftime(Sys.time(),t0,units="secs")) + best <- Lr$score; cum <- 0.0 + # Continuation budget starts AFTER L (matching route-3, whose arms also + # continue from L); cumulative wall is the continuation cost only. + set.seed(1000L + s) + kmark <- c(1,2,4,8,16); kmark <- kmark[kmark <= kmax] + for (k in seq_len(kmax)) { + r <- RandomTree(nt, root=TRUE); r$tip.label <- names(phy) + tt <- Sys.time(); rr <- tbrTo(r$edge, ds); cum <- cum + as.double(difftime(Sys.time(),tt,units="secs")) + if (rr$score < best) best <- rr$score + if (k %in% kmark) { + rows[[length(rows)+1L]] <- data.frame(dataset=nm, nTip=nt, seed=s, cycles=k, + arm="replicate", wall_s=round(cum,3), score=as.double(best)) + cat(sprintf("%-12s seed=%d k=%2d replicate best=%.0f wall=%.2fs\n", nm, s, k, best, cum)) + } + } + } +} +df <- do.call(rbind, rows); write.csv(df, outfile, row.names=FALSE) +cat(sprintf("\nWrote %d rows to %s\n", nrow(df), outfile)) diff --git a/dev/profiling/drift-exactness-route3-bench.R b/dev/profiling/drift-exactness-route3-bench.R new file mode 100644 index 000000000..a6408b8d5 --- /dev/null +++ b/dev/profiling/drift-exactness-route3-bench.R @@ -0,0 +1,61 @@ +#!/usr/bin/env Rscript +# Route-3 marginal-value experiment (advisor design). Question: in a wall-limited +# search, is a wall-second better spent on (cheap, exact) drift than on more of +# the incumbent diversifier (ratchet)? If yes, drift earns a place in fast/ +# wall-limited recipes that currently skip it -> worth re-tuning. +# +# From a SHARED per-seed local optimum (ts_tbr_search to convergence; scorer- +# independent), three continuations, each swept over a cycles knob (wall is the +# eval metric, per policy-in-replicates-not-seconds): +# ratchet : ts_ratchet_search (the incumbent marginal spend) +# drift_exact : ts_drift_search + TS_DRIFT_EXACT +# drift_union : ts_drift_search (union) (control: shows the unlock is exact's +# cheapness, not drift per se) +# Decision: if drift_exact's anytime frontier beats ratchet's over some wall +# range AND dominates drift_union -> install exact-drift in wall-limited recipes. +# If ratchet dominates everywhere -> route-3 no-go. +suppressMessages({ + library(TreeSearch, lib.loc = Sys.getenv("GATE_LIB", ".agent-hj")) + library(TreeTools) +}) +seeds <- seq_len(as.integer(Sys.getenv("GATE_SEEDS", "20"))) +cyc <- as.integer(strsplit(Sys.getenv("GATE_CYC", "1,2,4,8,16"), ",")[[1]]) +dnames <- strsplit(Sys.getenv("GATE_DATA", "Zhu2013,Zanol2014,Dikow2009"), ",")[[1]] +outfile <- Sys.getenv("GATE_OUT", "dev/profiling/drift-exactness-route3-bench.csv") + +fitchPhy <- function(p){m<-PhyDatToMatrix(p,ambigNA=FALSE); m[m=="-"]<-"?"; MatrixToPhyDat(m)} +loadPhy <- function(nm){ + for (p in c(file.path("data-raw", paste0(nm,".nex")), + file.path("dev/benchmarks", paste0(nm,".nex")))) + if (file.exists(p)) return(fitchPhy(ReadAsPhyDat(p))) + stop("not found: ", nm) +} +timed <- function(expr){t0<-Sys.time(); v<-force(expr); list(v=v, w=as.double(difftime(Sys.time(),t0,units="secs")))} + +rows <- list() +for (nm in dnames) { + phy <- loadPhy(nm); nt <- length(phy); at <- attributes(phy) + ds <- list(contrast=at$contrast, tip_data=matrix(unlist(phy,use.names=FALSE),nrow=nt,byrow=TRUE), + weight=at$weight, levels=at$levels) + for (s in seeds) { + set.seed(s); st <- RandomTree(nt, root=TRUE); st$tip.label <- names(phy) + L <- TreeSearch:::ts_tbr_search(st$edge, ds$contrast, ds$tip_data, ds$weight, ds$levels, maxHits=1L)$edge + for (k in cyc) { + arms <- list( + ratchet = function() TreeSearch:::ts_ratchet_search(L, ds$contrast, ds$tip_data, ds$weight, ds$levels, nCycles=k), + drift_exact = function(){Sys.setenv(TS_DRIFT_EXACT="1"); on.exit(Sys.unsetenv("TS_DRIFT_EXACT")) + TreeSearch:::ts_drift_search(L, ds$contrast, ds$tip_data, ds$weight, ds$levels, nCycles=k)}, + drift_union = function() TreeSearch:::ts_drift_search(L, ds$contrast, ds$tip_data, ds$weight, ds$levels, nCycles=k)) + for (arm in names(arms)) { + set.seed(1000L + s) + r <- timed(arms[[arm]]()) + rows[[length(rows)+1L]] <- data.frame(dataset=nm, nTip=nt, seed=s, cycles=k, + arm=arm, wall_s=round(r$w,3), score=as.double(r$v$score)) + cat(sprintf("%-12s seed=%d k=%2d %-12s score=%.0f wall=%.2fs\n", nm, s, k, arm, r$v$score, r$w)) + } + } + } +} +df <- do.call(rbind, rows) +write.csv(df, outfile, row.names=FALSE) +cat(sprintf("\nWrote %d rows to %s\n", nrow(df), outfile)) diff --git a/dev/profiling/drift-exactness-route3.md b/dev/profiling/drift-exactness-route3.md new file mode 100644 index 000000000..24f0104c7 --- /dev/null +++ b/dev/profiling/drift-exactness-route3.md @@ -0,0 +1,58 @@ +# Route-3: does cheap exact-drift earn a place in the recipe? — NO-GO + +Follow-on to `drift-exactness-gate.md`. The drift matched-wall gate showed +exact-drift wins the `auto`/wall-race regime; route-3 asked whether that's worth +**re-tuning recipes** to install/expand drift in wall-limited presets. Answer: +**no** — the marginal-value win does not survive into the full search ensemble. + +## Experiments (all Hamilton, EW, reuse lib-driftexact) + +1. **Marginal value** (`drift-exactness-route3-bench.R` + `-replicate-bench.R`): + from a shared per-seed local optimum, best marginal spend of a wall-second — + arms `ratchet` / `drift_exact` / `drift_union` / `replicate`. 3 gate datasets. +2. **Recipe whole-search** (`drift-exactness-recipe-bench.R`): whole `thorough` + search on **8 training-split MorphoBank keys** (65–135t, EW-recoded, sector + drift OFF to isolate main-tree drift), arms `none` / `union` / `exact`, + `maxReplicates` knob. TRAINING split only (sequestered; harness refuses + non-training keys). Raw CSVs in `route3-data/`; reanalyse `route3-analyze.R`. + +## Result 1 — marginal value: exact-drift is the BEST single continuation +At matched wall on all 3 datasets, `drift_exact` beats `ratchet`, `drift_union` +AND `replicate` (a fresh restart from a local-opt is the *worst* — it discards +progress). E.g. Dikow2009 @0.44s: exact 1607.85 < ratchet 1608.30 < union 1608.20 +< replicate 1610.00. So *in isolation*, a wall-second is best spent on exact-drift. + +## Result 2 — recipe whole-search: drift is REDUNDANT in the ensemble (both ends) +The isolated win does **not** translate. Across the 8 training keys (4 were +trivially converged = zero signal; effective N=4): + +- **Saturated (reps=8):** 4/8 exact-tie; non-ties sub-step noise split across + arms; **`none` ties on score and is faster** (project2124: none 33.2s vs union + 41.9s vs exact 35.6s). Adding drift costs ~10–25% wall for ~nil. +- **Tight budget (the regime route-3 is *about*; reps=2 / low W, non-flat keys):** + still no systematic exact win — project3617 `none` best, project4614 `union` + best (exact ≈ none), project2124 `exact` best, project3807 wash. exact-drift + beats no-drift on **1/4** keys; where drift helps it is as often union as exact. + +So the multistart + ratchet + sectorial + TBR ensemble already captures the score +that exact-drift finds as an isolated continuation; layering main-tree drift on +top is redundant at every budget and only adds wall. (Consistent with +`diversity-generation-gates`: main-tree drift OFF on the hard path.) + +## Verdict & what lands +- **NO recipe surgery.** Do not install/expand main-tree drift in any preset; + per `completeness-secondary-to-wallclock`, adding wall for no reach gain is a + disqualifier. exact-drift stays **opt-in** (`TS_DRIFT_EXACT`). +- **exact-drift's only residual value** is as the *cheaper* scorer *where drift + already runs* (thorough's sector godrift): ~equal reach, less wall — a minor + opt-in, not a default flip. Sector-godrift scorer itself was a wash/crossover + (`drift-exactness-gate.md`), reinforcing opt-in. +- **The genuine win from this line of work is `spr_search`** — exact promoted to + default (a real premature-convergence fix), see `drift-exactness-gate.md`. +- **D preset-refactor: not worth it.** No preset default should flip to + exact-drift, so the ABI/`SearchControl` plumbing has no consumer; the existing + `TS_DRIFT_EXACT` env flag suffices as the opt-in. +- **mbank 180t marginal-value curve: cancelled** (both original tasks TIMED OUT + at the 8h cap; the resubmit was cancelled once recipe-B settled the decision) — + it would only have re-confirmed the isolated marginal win, which is moot given + the ensemble redundancy. diff --git a/dev/profiling/drift-exactness-sector-bench.R b/dev/profiling/drift-exactness-sector-bench.R new file mode 100644 index 000000000..8720a6bf6 --- /dev/null +++ b/dev/profiling/drift-exactness-sector-bench.R @@ -0,0 +1,55 @@ +#!/usr/bin/env Rscript +# Sector-godrift isolation gate (route B). Full MaximizeParsimony "thorough" +# search with driftCycles=0 (main-tree drift OFF -- verified: gd=0 => 0 drift +# calls), so the ONLY drift is the in-sector godrift solve. Three arms: +# none : sectorGoDrift=0 (no sector drift at all -- does sector drift pay?) +# union : sectorGoDrift=25, union-of-finals scorer (current default) +# exact : sectorGoDrift=25, TS_DRIFT_EXACT exact directional scorer +# maxReplicates is the replicate-like policy knob; wall is the eval metric +# (policy-in-replicates-not-seconds). Same seed per arm => matched RNG. Writes +# (dataset, seed, reps, arm, wall_s, score) so the analysis reads score at +# matched wall. +suppressMessages({ + library(TreeSearch, lib.loc = Sys.getenv("GATE_LIB", ".agent-hj")) + library(TreeTools) +}) +seeds <- seq_len(as.integer(Sys.getenv("GATE_SEEDS", "15"))) +reps <- as.integer(strsplit(Sys.getenv("GATE_REPS", "1,2,4"), ",")[[1]]) +dnames <- strsplit(Sys.getenv("GATE_DATA", "Zhu2013,Zanol2014,Dikow2009"), ",")[[1]] +outfile <- Sys.getenv("GATE_OUT", "dev/profiling/drift-exactness-sector-bench.csv") +godrift <- as.integer(Sys.getenv("GATE_GODRIFT", "25")) + +fitchPhy <- function(p){m<-PhyDatToMatrix(p,ambigNA=FALSE); m[m=="-"]<-"?"; MatrixToPhyDat(m)} +loadPhy <- function(nm){ + for (p in c(file.path("data-raw", paste0(nm, ".nex")), + file.path("dev/benchmarks", paste0(nm, ".nex")), + file.path("dev/benchmarks/data", paste0(nm, ".nex")))) + if (file.exists(p)) return(fitchPhy(ReadAsPhyDat(p))) + stop("dataset not found: ", nm) +} + +rows <- list() +for (nm in dnames) { + phy <- loadPhy(nm); nt <- length(phy) + for (s in seeds) for (R in reps) { + for (arm in c("none","union","exact")) { + gd <- if (arm == "none") 0L else godrift + if (arm == "exact") Sys.setenv(TS_DRIFT_EXACT="1") else Sys.unsetenv("TS_DRIFT_EXACT") + set.seed(s) + t0 <- Sys.time() + res <- MaximizeParsimony(phy, strategy="thorough", driftCycles=0L, + sectorGoDrift=gd, maxReplicates=R, + nThreads=1L, verbosity=0L) + wall <- as.double(difftime(Sys.time(), t0, units="secs")) + Sys.unsetenv("TS_DRIFT_EXACT") + sc <- as.double(attr(res, "score")) + rows[[length(rows)+1L]] <- data.frame(dataset=nm, nTip=nt, seed=s, reps=R, + arm=arm, wall_s=round(wall,3), score=sc) + cat(sprintf("%-12s seed=%d reps=%d %-5s score=%.0f wall=%.2fs\n", + nm, s, R, arm, sc, wall)) + } + } +} +df <- do.call(rbind, rows) +write.csv(df, outfile, row.names=FALSE) +cat(sprintf("\nWrote %d rows to %s\n", nrow(df), outfile)) diff --git a/dev/profiling/drift-exactness-spr-smoke.R b/dev/profiling/drift-exactness-spr-smoke.R new file mode 100644 index 000000000..83c2902cb --- /dev/null +++ b/dev/profiling/drift-exactness-spr-smoke.R @@ -0,0 +1,38 @@ +#!/usr/bin/env Rscript +# Validate the spr_search exact path (TS_SPR_EXACT): the returned score must +# equal an INDEPENDENT full Fitch re-score of the returned tree (wiring), and the +# hill-climb must improve on the start. Also contrasts union vs exact reach -- +# spr_search is a pure hill-climb with no compensating exact-TBR phase, so the +# union over-count (which trips the `dominated` gate at ts_search.cpp:379 and +# skips improving moves) causes premature convergence the exact path avoids. +suppressMessages({ + library(TreeSearch, lib.loc = Sys.getenv("GATE_LIB", ".agent-hj")) + library(TreeTools) +}) +fitchPhy <- function(p) { + m <- PhyDatToMatrix(p, ambigNA = FALSE); m[m == "-"] <- "?"; MatrixToPhyDat(m) +} +phy <- fitchPhy(ReadAsPhyDat("data-raw/Zhu2013.nex")) +at <- attributes(phy) +ds <- list(contrast = at$contrast, + tip_data = matrix(unlist(phy, use.names = FALSE), + nrow = length(phy), byrow = TRUE), + weight = at$weight, levels = at$levels) +indep <- function(edge) TreeSearch:::ts_fitch_score(edge, ds$contrast, ds$tip_data, + ds$weight, ds$levels, min_steps = integer(0), concavity = -1) +for (s in 1:5) { + set.seed(s) + st <- RandomTree(length(phy), root = TRUE); st$tip.label <- names(phy) + start_len <- indep(st$edge) + for (ex in c(FALSE, TRUE)) { + if (ex) Sys.setenv(TS_SPR_EXACT = "1") else Sys.unsetenv("TS_SPR_EXACT") + set.seed(100 + s) + r <- TreeSearch:::ts_spr_search(st$edge, ds$contrast, ds$tip_data, + ds$weight, ds$levels, maxHits = 5L) + Sys.unsetenv("TS_SPR_EXACT") + chk <- indep(r$edge) + cat(sprintf("seed=%d %-5s start=%.0f returned=%.0f indep=%.0f MATCH=%s improved=%s\n", + s, if (ex) "exact" else "union", start_len, r$score, chk, + abs(r$score - chk) < 1e-6, r$score <= start_len)) + } +} diff --git a/dev/profiling/exactness-gate.R b/dev/profiling/exactness-gate.R new file mode 100644 index 000000000..eb5c97483 --- /dev/null +++ b/dev/profiling/exactness-gate.R @@ -0,0 +1,309 @@ +# Exactness gate harness (reproducible). See exactness-gate.md for the verdict. +# Run: Rscript dev/profiling/exactness-gate.R +# Requires: TreeSearch, phangorn, ape, TreeTools. Pure R; independent of the C++ engine. + +# Helper library: from-scratch bitmask Fitch (down + up), oracle wrappers, +# brute-force finals, prune/regraft. Equal-weights Fitch, unordered characters. +# States are 0-indexed; bitmask for state s is bit (s). Missing/"?"/"-" = all bits. + +suppressMessages({ + library(TreeSearch) + library(phangorn) + library(ape) +}) + +fullMask <- function(nStates) bitwShiftL(1L, nStates) - 1L + +stateToMask <- function(x, nStates) { + if (is.na(x) || x == "?" || x == "-") return(fullMask(nStates)) + bitwShiftL(1L, as.integer(x)) +} + +# charMat: character matrix, rows = tips (rownames = tip labels), cols = chars. +masksFromChars <- function(charMat, nStates) { + m <- matrix(0L, nrow(charMat), ncol(charMat)) + for (i in seq_len(nrow(charMat))) + for (j in seq_len(ncol(charMat))) + m[i, j] <- stateToMask(charMat[i, j], nStates) + m +} + +# Build a tip-mask matrix whose ROW i corresponds to NODE i of `tree` +# (tree$tip.label[i]), aligning arbitrary charMat row order to the tree's tip +# numbering. ape::rtree() permutes tip labels, so this alignment is essential. +alignedTipMask <- function(tree, charMat, nStates) { + stopifnot(all(tree$tip.label %in% rownames(charMat))) + masksFromChars(charMat[tree$tip.label, , drop = FALSE], nStates) +} +# Aligned integer states (0-indexed; NA for missing "?"/"-"), row i = node i. +alignedStates <- function(tree, charMat) { + cm <- charMat[tree$tip.label, , drop = FALSE] + out <- suppressWarnings(matrix(as.integer(cm), nrow(cm), ncol(cm))) + out +} + +getRoot <- function(tree) setdiff(unique(tree$edge[, 1]), unique(tree$edge[, 2])) +childrenOf <- function(edge, node) edge[edge[, 1] == node, 2] +parentOf <- function(edge, node) edge[edge[, 2] == node, 1] +siblingOf <- function(edge, node) { + p <- parentOf(edge, node) + setdiff(childrenOf(edge, p), node) +} + +# Preorder node list (parents strictly before descendants). +preorderNodes <- function(tree) { + root <- getRoot(tree); edge <- tree$edge + out <- integer(0); stack <- root + while (length(stack)) { + nd <- stack[length(stack)]; stack <- stack[-length(stack)] + out <- c(out, nd) + ch <- childrenOf(edge, nd) + if (length(ch)) stack <- c(stack, ch) + } + out +} +# Postorder internal nodes (all children before their parent). +postorderInternal <- function(tree) { + pre <- preorderNodes(tree) + r <- rev(pre) + r[r > length(tree$tip.label)] +} + +combineMask <- function(a, b) { + inter <- bitwAnd(a, b) + ifelse(inter != 0L, inter, bitwOr(a, b)) +} + +# Fitch down-pass: returns prelim (per node) and total length. Handles >2 +# children by sequential folding (exact for binary; folding for polytomies). +fitchDown <- function(tree, tipMask) { + nTip <- length(tree$tip.label) + nNode <- nTip + tree$Nnode + nChar <- ncol(tipMask) + prelim <- matrix(0L, nNode, nChar) + prelim[seq_len(nTip), ] <- tipMask + len <- integer(nChar) + for (nd in postorderInternal(tree)) { + ch <- childrenOf(tree$edge, nd) + acc <- prelim[ch[1], ] + if (length(ch) >= 2) for (j in 2:length(ch)) { + b <- prelim[ch[j], ] + inter <- bitwAnd(acc, b) + hit <- inter != 0L + acc <- ifelse(hit, inter, bitwOr(acc, b)) + len <- len + as.integer(!hit) + } + prelim[nd, ] <- acc + } + list(prelim = prelim, length = sum(len), perChar = len) +} + +# Fitch up-pass (rooted, degree-2 root): up[nd] = message into nd from parent +# side; final[nd] = combine(prelim[nd], up[nd]). root final = prelim (unused +# for edges). +fitchUp <- function(tree, prelim) { + nTip <- length(tree$tip.label) + nNode <- nTip + tree$Nnode + nChar <- ncol(prelim) + edge <- tree$edge + root <- getRoot(tree) + up <- matrix(0L, nNode, nChar) + final <- matrix(0L, nNode, nChar) + final[root, ] <- prelim[root, ] + for (nd in preorderNodes(tree)) { + if (nd == root) next + A <- parentOf(edge, nd) + sib <- siblingOf(edge, nd) + if (A == root) { + # degree-2 root: message is sibling's prelim + upnd <- prelim[sib[1], ] + if (length(sib) > 1) for (k in 2:length(sib)) upnd <- combineMask(upnd, prelim[sib[k], ]) + } else { + upnd <- combineMask(up[A, ], prelim[sib[1], ]) + if (length(sib) > 1) for (k in 2:length(sib)) upnd <- combineMask(upnd, prelim[sib[k], ]) + } + up[nd, ] <- upnd + final[nd, ] <- combineMask(prelim[nd, ], upnd) + } + list(up = up, final = final) +} + +# Engine's SIMPLIFIED up-pass (replicates uppass_node / fitch_uppass in +# src/ts_fitch.cpp). Top-down: final(root) = prelim(root); for each other node +# final(node) = final(parent) & prelim(node) if non-empty, else prelim(node). +# Never unions -> engine_final(node) is always a SUBSET of prelim(node). +fitchUpEngine <- function(tree, prelim) { + nTip <- length(tree$tip.label) + nNode <- nTip + tree$Nnode + nChar <- ncol(prelim) + edge <- tree$edge + root <- getRoot(tree) + final <- matrix(0L, nNode, nChar) + final[root, ] <- prelim[root, ] + for (nd in preorderNodes(tree)) { + if (nd == root) next + A <- parentOf(edge, nd) + isect <- bitwAnd(final[A, ], prelim[nd, ]) + final[nd, ] <- ifelse(isect != 0L, isect, prelim[nd, ]) + } + list(final = final) +} + +# Regraft subtree S onto the edge above node D of `base` (a new node w splits +# that edge; w's children = D and S's root). Explicit edge-matrix surgery, then +# normalise numbering via write/read newick. Branch lengths dropped (Fitch is +# length-invariant). Returns a phylo. +regraft <- function(base, S, D) { + nTb <- length(base$tip.label); nTs <- length(S$tip.label) + Nb <- base$Nnode; Ns <- S$Nnode + nT <- nTb + nTs + mapBase <- function(x) ifelse(x <= nTb, x, nT + (x - nTb)) + mapS <- function(x) ifelse(x <= nTs, nTb + x, nT + Nb + (x - nTs)) + w <- nT + Nb + Ns + 1L + eb <- base$edge; es <- S$edge + ebm <- cbind(mapBase(eb[, 1]), mapBase(eb[, 2])) + esm <- cbind(mapS(es[, 1]), mapS(es[, 2])) + rsMapped <- mapS(getRoot(S)) + Dm <- mapBase(D); Am <- mapBase(parentOf(eb, D)) + ebm <- ebm[!(ebm[, 1] == Am & ebm[, 2] == Dm), , drop = FALSE] + newEdges <- rbind(ebm, esm, c(Am, w), c(w, Dm), c(w, rsMapped)) + tr <- structure(list(edge = newEdges, + tip.label = c(base$tip.label, S$tip.label), + Nnode = Nb + Ns + 1L), class = "phylo") + ape::read.tree(text = ape::write.tree(tr)) +} + +# Oracle: build phyDat and score with TreeSearch::TreeLength (Fitch, EW). +makePhyDat <- function(charMat, nStates) { + phangorn::phyDat(charMat, type = "USER", levels = as.character(0:(nStates - 1))) +} +treeLen <- function(tree, pd) as.numeric(TreeSearch::TreeLength(tree, pd)) + +# Brute-force finals for a SINGLE unambiguous character. tipState: integer +# vector length nTip (single state per tip, 0-indexed). Returns list: +# optLen, finalMask (per node bitmask of states appearing in some optimum). +bruteFinals <- function(tree, tipState, nStates) { + nTip <- length(tree$tip.label) + internal <- (nTip + 1):(nTip + tree$Nnode) + edge <- tree$edge + nInt <- length(internal) + # enumerate all assignments of states to internal nodes + grid <- as.matrix(expand.grid(rep(list(0:(nStates - 1)), nInt))) + fullNode <- integer(nTip + tree$Nnode) + fullNode[seq_len(nTip)] <- tipState + lens <- integer(nrow(grid)) + for (g in seq_len(nrow(grid))) { + fullNode[internal] <- grid[g, ] + lens[g] <- sum(fullNode[edge[, 1]] != fullNode[edge[, 2]]) + } + optLen <- min(lens) + optRows <- which(lens == optLen) + finalMask <- integer(nTip + tree$Nnode) + finalMask[seq_len(nTip)] <- bitwShiftL(1L, tipState) + for (k in seq_along(internal)) { + nd <- internal[k] + states <- unique(grid[optRows, k]) + finalMask[nd] <- Reduce(bitwOr, bitwShiftL(1L, states), 0L) + } + list(optLen = optLen, finalMask = finalMask) +} + +# ============================ EXPERIMENT ============================ + +disjointCount <- function(X, S) sum(bitwAnd(X, S) == 0L) +scoreSub <- function(tr, cm, nStates) treeLen(tr, makePhyDat(cm[tr$tip.label,,drop=FALSE], nStates)) + +# single-tip "subtree": root (Nnode=1) with the one tip as its only child +oneTipTree <- function(lbl) structure(list(edge = matrix(c(2L,1L),1,2), + tip.label = lbl, Nnode = 1L), class = "phylo") + +runClip <- function(tr, cm, nStates, s) { + nTipT <- length(tr$tip.label) + if (s <= nTipT) { # clip a single leaf + Stips <- tr$tip.label[s]; S <- oneTipTree(Stips) + } else { + S <- ape::extract.clade(tr, s); Stips <- S$tip.label + } + if (length(Stips) > nTipT - 3) return(NULL) + base <- ape::drop.tip(tr, Stips, collapse.singles = TRUE) + if (length(base$tip.label) < 3) return(NULL) + + Smask <- alignedTipMask(S, cm, nStates) + Sdn <- fitchDown(S, Smask) + X <- Sdn$prelim[getRoot(S), ] + Lwithin <- Sdn$length + Lbase <- scoreSub(base, cm, nStates) + + bMask <- alignedTipMask(base, cm, nStates) + bDn <- fitchDown(base, bMask) + E <- fitchUp(base, bDn$prelim)$final # edge sets (engine EXACT path) + Fe <- fitchUpEngine(base, bDn$prelim)$final # engine SIMPLIFIED final_ + root <- getRoot(base) + cands <- setdiff(seq_len(length(base$tip.label) + base$Nnode), root) + + rows <- vector("list", length(cands)); k <- 0 + for (D in cands) { + A <- parentOf(base$edge, D) + rec <- tryCatch(regraft(base, S, D), error = function(e) NULL) + if (is.null(rec)) next + added <- as.integer(round(scoreSub(rec, cm, nStates) - Lbase - Lwithin)) + k <- k + 1 + rows[[k]] <- data.frame(nStates = nStates, clipSize = length(Stips), + baseTips = length(base$tip.label), added = added, + pEdge = disjointCount(X, E[D, ]), + pUnionSimp = disjointCount(X, bitwOr(Fe[A, ], Fe[D, ])), + pUnionEdge = disjointCount(X, bitwOr(E[A, ], E[D, ])), + pSingleSimp = disjointCount(X, Fe[D, ])) + } + if (!k) return(NULL) + do.call(rbind, rows[seq_len(k)]) +} + +randChars <- function(nTip, nChar, nStates, pMissing = 0) { + m <- matrix(as.character(sample(0:(nStates-1), nTip*nChar, replace = TRUE)), nTip, nChar) + if (pMissing > 0) { miss <- matrix(runif(nTip*nChar) < pMissing, nTip, nChar); m[miss] <- "?" } + rownames(m) <- paste0("t", seq_len(nTip)); m +} +cladeSize <- function(tr, node) if (node <= length(tr$tip.label)) 1L else length(ape::extract.clade(tr, node)$tip.label) + +runRegime <- function(label, nStates, pMissing, nTrees, seed) { + set.seed(seed); all <- list() + for (r in seq_len(nTrees)) { + nTip <- sample(8:13, 1); nChar <- sample(8:16, 1) + cm <- randChars(nTip, nChar, nStates, pMissing) + tr <- ape::rtree(nTip, tip.label = rownames(cm)) + internal <- (nTip+1):(nTip+tr$Nnode); root <- getRoot(tr) + sizes <- sapply(internal, function(n) cladeSize(tr, n)) + goodInt <- internal[internal != root & sizes >= 2 & sizes <= nTip - 3] + clips <- c(if (length(goodInt)) sample(goodInt, min(4, length(goodInt))), + sample(seq_len(nTip), 2)) # + 2 leaf clips + for (s in clips) { + res <- tryCatch(runClip(tr, cm, nStates, s), error = function(e) NULL) + if (!is.null(res)) all[[length(all)+1]] <- res + } + } + df <- do.call(rbind, all); df$regime <- label; df +} + +reportRegime <- function(df) { + cat(sprintf("\n=== %s (nStates=%d, %d candidates, clipSizes %d..%d) ===\n", + df$regime[1], df$nStates[1], nrow(df), min(df$clipSize), max(df$clipSize))) + show <- function(tag, pred) { + e <- df$added - pred + cat(sprintf(" %-22s err(added-pred): min=%d max=%d | over(<0)=%d under(>0)=%d exact(0)=%d (%.1f%% exact)\n", + tag, min(e), max(e), sum(e<0), sum(e>0), sum(e==0), 100*mean(e==0))) + } + show("P1 edge-set(exact)", df$pEdge) + show("P2 union-simplified", df$pUnionSimp) # DEPLOYED fitch_indirect_length + show("P3 union-edgesets", df$pUnionEdge) + show("P4 single-simplified", df$pSingleSimp) +} + +cat("Running 4 regimes...\n") +res <- list( + bin = runRegime("binary-noNA", 2, 0.00, 30, 101), + binNA = runRegime("binary-15NA", 2, 0.15, 30, 102), + mult = runRegime("multistate4-noNA", 4, 0.00, 30, 103), + multNA = runRegime("multistate4-20NA", 4, 0.20, 30, 104)) +for (df in res) reportRegime(df) +cat("\nSaved results.rds\n") diff --git a/dev/profiling/exactness-gate.md b/dev/profiling/exactness-gate.md new file mode 100644 index 000000000..3aac2478c --- /dev/null +++ b/dev/profiling/exactness-gate.md @@ -0,0 +1,163 @@ +# Exactness gate: is fresh rooted union-of-finals an EXACT Fitch insertion cost? + +**Question (gates a perf refactor).** TS's TBR reconnection scoring pays an +expensive *directional edge-set* precompute (`compute_insertion_edge_sets`, +`src/ts_fitch.cpp:521`) and then, per candidate edge (A,D), counts characters +where the clipped subtree's prelim set is disjoint from `edge_set[D]`. A cheaper +candidate is **union-of-finals**: cost at edge (A,D) = #chars where +`clipRootStates ∩ (final(A) | final(D)) = ∅` (`fitch_indirect_length`, +`src/ts_fitch.cpp:397`). Prior work abandoned union-of-finals after it +mis-counted, but possibly only because finals were stale or the framework was +unrooted. **Is a CLEAN, freshly-recomputed ROOTED union-of-finals insertion cost +EXACT, or only a bound?** + +## Verdict (three-way) + +Tested empirically against full-rescore ground truth (`TreeSearch::TreeLength`) +over **6066 reconnection candidates** spanning binary/multistate, missing-data, +clip sizes 1–10, and multiple rootings. Fresh Fitch passes recomputed on the +clipped tree throughout — never incremental/stale. + +| Predictor | What it is | Result | Verdict | +|---|---|---|---| +| **P1** `edge_set[D] = combine(prelim[D], up[D])` | engine's **exact** path, per-child directional edge set | signed error `added − pred = 0` for **6066 / 6066** | **EXACT** (rooting-invariant) | +| **P3** union of **true** edge sets `E(A) \| E(D)` | union-of-finals using the *true* directional finals | signed error ∈ **{0,+1,+2,+3,+4}**, never < 0 | **SOUND LOWER BOUND** (under-counts; never exact) | +| **P2** union of the engine's **stored** `final_` | the **deployed** `fitch_indirect_length` reads `tree.final_` | signed error ∈ **{−5,−4,−3,−2,−1,0}**, never > 0 | **UNSAFE — strict OVER-count** (upper bound) | + +**Bottom line: a cheap "root like TNT → exact union-of-finals" refactor is a +MIRAGE.** Union-of-finals is *never* exact for EW-Fitch insertion cost — not even +on binary unambiguous data, and not under any rooting. At best it is a sound +*lower* bound, and only when computed over the *true* directional edge sets +`combine(prelim, up)` — which cost exactly the same directional up-pass the +refactor was trying to avoid. The only exact insertion cost is the per-child +edge set `E(D)`, which the engine already computes. + +Worse, the union variant that is actually **deployed** today +(`fitch_indirect_length`, which reads the incrementally-maintained +`tree.final_`) is **not even a sound bound: it OVER-counts** the true insertion +cost (by up to +5 here), because `tree.final_` is a *simplified* final set. + +## Why the deployed union over-counts (mechanism, not staleness) + +The stored `final_` (see `uppass_node`, `src/ts_fitch.cpp:41-70`) uses a +**simplified** rule: + +``` +final(root) = prelim(root) +final(node) = final(parent) ∩ prelim(node) if non-empty + = prelim(node) otherwise // NEVER unions +``` + +Because it never performs Fitch's *union* step, `final_(node) ⊆ prelim(node)` +always — it is generally a **strict subset** of the true MPR/edge-set final. +Union of two too-small sets is too small ⇒ more characters look "disjoint" ⇒ +over-count. Concrete minimal case found in the sweep (binary, fresh finals): + +``` +char: clipRootStates X = {1}, prelim(D) = {0} + true edge set E(D) = combine(prelim(D), up(D)) = {0,1} (up-pass unions in state 1) + -> X ∩ E(D) = {1} != empty -> TRUE added step = 0 + deployed simplified finals: final_(A) = {0}, final_(D) = {0}, union = {0} + -> X ∩ {0} = empty -> DEPLOYED union predicts step = 1 (OVER-COUNT) +``` + +Finals here are freshly recomputed, so this is a **formula limitation of the +simplified up-pass**, not a staleness artifact. The theoretical "union +under-counts" argument (a union of finals is a superset of the edge set, so it +only *adds* available states) is correct **only for true finals** (that is what +P3 confirms); it does **not** hold for the simplified `final_` the engine stores +and the union screen reads. + +### Reconciliation with the code comment +`fitch_indirect_length`'s comment claims the union "UNDER-counts the true +insertion cost (never over-counts)." That is **false for the deployed code**: it +holds for a union of *true* edge sets (P3), but `tree.final_` is the simplified +final, and the deployed union over-counts (P2). Any caller that treats +`fitch_indirect_length` as an *admissible lower bound* (e.g. to prune/skip a +candidate whose bound ≥ current best) is **unsound** — it can discard the true +optimum. It is only safe in its documented use as an *approximate ranker* +followed by exact re-scoring, and even then it can mis-rank (demote a genuinely +good reconnection). + +## Equality-on-binary (the target-dataset question) + +project5432 is mostly binary, so the key sub-question was: does union == exact on +unambiguous binary characters (where one might hope finals are singletons)? +**No.** Internal Fitch sets are frequently non-singleton even with unambiguous +binary tips, and both union variants fail on binary: + +- Binary regimes (2896 candidates): **P1 exact 100%**; P2 (deployed) exact only + 65.2% (**over-counts 34.8%**); P3 exact 72.4% (**under-counts 27.6%**). + +## Effect of conditions + +Exactness of **P1 is unconditional** (100% across every cell). The union +predictors are never exact and degrade with alphabet size: + +| regime | candidates | P1 exact | P2 (deployed) exact / over | P3 exact / under | +|---|---|---|---|---| +| binary, no missing | 1410 | 100% | 68.5% / over ≤3 | 70.8% / under ≤4 | +| binary, 15% missing | 1486 | 100% | 62.0% / over ≤4 | 73.9% / under ≤3 | +| 4-state, no missing | 1564 | 100% | 51.8% / over ≤5 | 60.6% / under ≤4 | +| 4-state, 20% missing | 1606 | 100% | 42.8% / over ≤5 | 67.1% / under ≤3 | + +- **Rooting:** `E(D)` is a property of the edge (bipartition), so P1 is + rooting-invariant — verified exact under 3–4 distinct rootings of the same + clipped tree across 8 further trees. The union predictors are rooting-*dependent* + (magnitudes shift), but P2 still over-counts and P3 still under-counts under + every rooting tried. +- **Clip size / leaf-vs-clade:** single-leaf (SPR) and multi-tip clade (TBR) + reinsertions both included; no effect on the verdict. +- **Multifurcations:** out of scope — the engine reconnects onto strictly binary + trees during search; the exact-vs-bound question is about binary base trees. + +## Method & reproducibility + +Pure R, independent of the C++ engine (guarantees genuinely fresh finals and an +independent oracle). Files (this dir): `exactness-gate.R` (self-contained +harness). Pipeline: + +1. **Ground truth = full re-score.** For rooted tree T, clip clade S at node s → + base T′. For each candidate edge (parent(D), D) in T′, build the reconnected + tree by explicit edge-matrix surgery (`regraft`) and score with + `TreeSearch::TreeLength`. `added(e) = L(reconnected) − L(T′) − L_within(S)`. +2. **Predictions from FRESH passes on T′.** `prelim` = my Fitch down-pass; + `E = combine(prelim, up)` with `up[D] = combine(up[parent], prelim[sibling])` + (root child ⇒ `prelim[other child]`) — this exactly reproduces + `compute_insertion_edge_sets`. `Fe` = the engine's simplified `final_` rule. + `X` = fresh prelim at S's root. +3. Compare `added` to P1 `|X ∩ E(D)|=∅`, P2 `|X ∩ (Fe(A)|Fe(D))|=∅`, + P3 `|X ∩ (E(A)|E(D))|=∅`. + +**Validation of the pipeline itself (so the ground truth is trustworthy):** +- My Fitch down-pass length == `TreeSearch::TreeLength` == `phangorn::fitch` + (after fixing a tip-label/node-index alignment: `ape::rtree` permutes tip + labels — the oracle matches by label, so tip masks must be aligned to + `tree$tip.label`, not to data row order). +- `regraft` well-formed and both scorers (TreeSearch + phangorn) agree on + **3902/3902** reconnected trees; regrafting at the original clip location + recovers L(T) on **307/307** checks. +- P1 == full-rescore on all 6066 candidates independently cross-validates the + edge-set formula, the regraft surgery, and the decomposition simultaneously. + +**Caveat on "true MPR final ≠ edge set."** A node's MPR set (states optimal at +the node) is *not* equal to `combine(prelim, up)` in general (the naive "one +extra step" reasoning fails for node states — forcing a cherry's ancestor to the +wrong state can cost +2). But the object relevant to *insertion at an edge* is +the edge set `combine(prelim(D), up(D))`, and inserting a subtree adds ≤ 1 +step/char; P1's 100% exactness confirms the edge set (not the node MPR set) is +the right and exact quantity. + +## Implication for the refactor + +- **Do NOT** replace the exact per-child edge-set scoring with a union-of-finals + screen expecting exactness — it is a bound at best, under every rooting, + including on binary data. +- **Do NOT** rely on the deployed `fitch_indirect_length` as an admissible lower + bound for pruning; it over-counts and can reject the optimum. If a sound lower + bound is wanted, it must union the *true* edge sets `combine(prelim,up)` (P3) — + but that requires the same directional up-pass as the exact path, so it buys no + precompute savings. +- The exact, cheap-per-candidate quantity is the per-child edge set `E(D)`, which + the engine already precomputes. The genuine perf lever is making that + precompute cheaper, not swapping in union-of-finals. diff --git a/dev/profiling/route3-analyze.R b/dev/profiling/route3-analyze.R new file mode 100644 index 000000000..9b6b7abe8 --- /dev/null +++ b/dev/profiling/route3-analyze.R @@ -0,0 +1,34 @@ +dir <- "dev/profiling/route3-data" +sAtW <- function(d,W){s<-unique(d$seed); v<-sapply(s,function(x){r<-d[d$seed==x & d$wall_s<=W,]; if(nrow(r)==0) NA else min(r$score)}); mean(v,na.rm=TRUE)} + +cat("################ RECIPE B (training MorphoBank, whole-search thorough) ################\n") +cat("arms: none (driftCycles=0) / union / exact ; knob=maxReplicates{2,4,8}; matched wall\n") +rk <- Sys.glob(file.path(dir,"recipe_project*.csv")) +wintab <- c(none=0,union=0,exact=0,tie=0) +for(f in rk){ d<-read.csv(f); key<-sub("_.*","",sub("recipe_","",basename(f))) + arms<-c("none","union","exact"); mw<-max(d$wall_s); Ws<-signif(mw*c(.15,.35,.6,1),2) + # winner at the LARGEST matched wall (most budget) per key + vE<-sAtW(d[d$arm=="exact",],mw); vU<-sAtW(d[d$arm=="union",],mw); vN<-sAtW(d[d$arm=="none",],mw) + best<-which.min(c(none=vN,union=vU,exact=vE)); w<-names(best) + if(max(abs(c(vN,vU,vE)-min(c(vN,vU,vE))))<1e-9) w<-"tie" + wintab[w]<-wintab[w]+1 + cat(sprintf("\n%-14s nTip=%d @maxwall %.1fs: none=%.1f union=%.1f exact=%.1f -> %s\n", + key, d$nTip[1], mw, vN,vU,vE, toupper(w))) + # per-arm mean score at reps=8 and mean wall + for(a in arms){da<-d[d$arm==a & d$reps==max(d$reps),]; cat(sprintf(" %-5s reps=8 mean score=%.1f wall=%.1fs\n",a,mean(da$score),mean(da$wall_s)))} +} +cat(sprintf("\n== RECIPE win-tally @max budget across %d keys: none=%d union=%d exact=%d tie=%d ==\n", + length(rk), wintab["none"],wintab["union"],wintab["exact"],wintab["tie"])) + +cat("\n\n################ ROUTE-3 + REPLICATE (marginal value from shared local-opt) ################\n") +cat("arms: ratchet / drift_exact / drift_union / replicate ; knob=cycles or restarts; matched wall\n") +for(nm in c("Zhu2013","Zanol2014","Dikow2009")){ + r3<-file.path(dir,sprintf("route3_%s.csv",nm)); rp<-file.path(dir,sprintf("repl_%s.csv",nm)) + if(!file.exists(r3)||!file.exists(rp)) next + d<-rbind(read.csv(r3), read.csv(rp)); arms<-c("ratchet","drift_exact","drift_union","replicate") + mw<-min(sapply(arms,function(a) max(d$wall_s[d$arm==a]))); Ws<-signif(mw*c(.2,.5,1),2) + cat(sprintf("\n== %s : matched-WALL frontier (mean score) ==\n",nm)) + cat(sprintf(" %6s %9s %11s %11s %10s best\n","wall","ratchet","drift_exact","drift_union","replicate")) + for(W in Ws){ v<-sapply(arms,function(a) sAtW(d[d$arm==a,],W)) + cat(sprintf(" %6.2f %9.2f %11.2f %11.2f %10.2f %s\n",W,v["ratchet"],v["drift_exact"],v["drift_union"],v["replicate"],names(which.min(v)))) } +} diff --git a/dev/profiling/route3-data/recipe_project2124.csv b/dev/profiling/route3-data/recipe_project2124.csv new file mode 100644 index 000000000..6973606a0 --- /dev/null +++ b/dev/profiling/route3-data/recipe_project2124.csv @@ -0,0 +1,55 @@ +"key","nTip","seed","reps","arm","wall_s","score" +"project2124",81,1,2,"none",8.647,1979 +"project2124",81,1,2,"union",8.867,1978 +"project2124",81,1,2,"exact",8.016,1976 +"project2124",81,1,4,"none",14.759,1977 +"project2124",81,1,4,"union",18.035,1978 +"project2124",81,1,4,"exact",16.441,1976 +"project2124",81,1,8,"none",24.017,1976 +"project2124",81,1,8,"union",37.998,1976 +"project2124",81,1,8,"exact",36.015,1976 +"project2124",81,2,2,"none",12.015,1977 +"project2124",81,2,2,"union",12.238,1976 +"project2124",81,2,2,"exact",9.043,1977 +"project2124",81,2,4,"none",20.036,1977 +"project2124",81,2,4,"union",25.308,1976 +"project2124",81,2,4,"exact",17.456,1977 +"project2124",81,2,8,"none",33.276,1976 +"project2124",81,2,8,"union",45.464,1976 +"project2124",81,2,8,"exact",33.583,1976 +"project2124",81,3,2,"none",10.548,1976 +"project2124",81,3,2,"union",10.406,1977 +"project2124",81,3,2,"exact",9.432,1977 +"project2124",81,3,4,"none",20.981,1976 +"project2124",81,3,4,"union",20.17,1977 +"project2124",81,3,4,"exact",18.129,1977 +"project2124",81,3,8,"none",33.691,1976 +"project2124",81,3,8,"union",41.859,1977 +"project2124",81,3,8,"exact",37.408,1977 +"project2124",81,4,2,"none",13.956,1976 +"project2124",81,4,2,"union",13.018,1976 +"project2124",81,4,2,"exact",12.114,1976 +"project2124",81,4,4,"none",21.342,1976 +"project2124",81,4,4,"union",20.29,1976 +"project2124",81,4,4,"exact",22.131,1976 +"project2124",81,4,8,"none",44.768,1976 +"project2124",81,4,8,"union",44.56,1976 +"project2124",81,4,8,"exact",40.28,1976 +"project2124",81,5,2,"none",5.843,1980 +"project2124",81,5,2,"union",11.683,1978 +"project2124",81,5,2,"exact",9.403,1981 +"project2124",81,5,4,"none",14.346,1978 +"project2124",81,5,4,"union",21.992,1976 +"project2124",81,5,4,"exact",18.037,1976 +"project2124",81,5,8,"none",27.409,1978 +"project2124",81,5,8,"union",42.471,1976 +"project2124",81,5,8,"exact",34.341,1976 +"project2124",81,6,2,"none",9.009,1978 +"project2124",81,6,2,"union",7.877,1980 +"project2124",81,6,2,"exact",7.179,1980 +"project2124",81,6,4,"none",19.661,1978 +"project2124",81,6,4,"union",18.662,1976 +"project2124",81,6,4,"exact",17.259,1976 +"project2124",81,6,8,"none",35.865,1976 +"project2124",81,6,8,"union",38.995,1976 +"project2124",81,6,8,"exact",31.965,1976 diff --git a/dev/profiling/route3-data/recipe_project3602.csv b/dev/profiling/route3-data/recipe_project3602.csv new file mode 100644 index 000000000..fa2d0d092 --- /dev/null +++ b/dev/profiling/route3-data/recipe_project3602.csv @@ -0,0 +1,55 @@ +"key","nTip","seed","reps","arm","wall_s","score" +"project3602",105,1,2,"none",6.58,918 +"project3602",105,1,2,"union",5.897,918 +"project3602",105,1,2,"exact",5.139,918 +"project3602",105,1,4,"none",11.195,917 +"project3602",105,1,4,"union",10.369,918 +"project3602",105,1,4,"exact",9.284,917 +"project3602",105,1,8,"none",21.288,917 +"project3602",105,1,8,"union",20.884,917 +"project3602",105,1,8,"exact",19.368,917 +"project3602",105,2,2,"none",5.872,918 +"project3602",105,2,2,"union",4.859,919 +"project3602",105,2,2,"exact",4.192,919 +"project3602",105,2,4,"none",11.675,917 +"project3602",105,2,4,"union",10.242,917 +"project3602",105,2,4,"exact",8.917,917 +"project3602",105,2,8,"none",21.306,917 +"project3602",105,2,8,"union",23.132,917 +"project3602",105,2,8,"exact",20.486,917 +"project3602",105,3,2,"none",5.106,917 +"project3602",105,3,2,"union",6.964,917 +"project3602",105,3,2,"exact",5.728,917 +"project3602",105,3,4,"none",10.221,917 +"project3602",105,3,4,"union",13.639,917 +"project3602",105,3,4,"exact",11.225,917 +"project3602",105,3,8,"none",19.644,917 +"project3602",105,3,8,"union",24.306,917 +"project3602",105,3,8,"exact",21.963,917 +"project3602",105,4,2,"none",5.46,917 +"project3602",105,4,2,"union",6.05,917 +"project3602",105,4,2,"exact",4.481,917 +"project3602",105,4,4,"none",9.505,917 +"project3602",105,4,4,"union",11.445,917 +"project3602",105,4,4,"exact",10.704,917 +"project3602",105,4,8,"none",17.202,917 +"project3602",105,4,8,"union",22.408,917 +"project3602",105,4,8,"exact",21.031,917 +"project3602",105,5,2,"none",3.685,917 +"project3602",105,5,2,"union",4.119,917 +"project3602",105,5,2,"exact",3.625,917 +"project3602",105,5,4,"none",6.894,917 +"project3602",105,5,4,"union",9.378,917 +"project3602",105,5,4,"exact",8.158,917 +"project3602",105,5,8,"none",16.38,917 +"project3602",105,5,8,"union",20.472,917 +"project3602",105,5,8,"exact",17.917,917 +"project3602",105,6,2,"none",4.033,917 +"project3602",105,6,2,"union",4.727,917 +"project3602",105,6,2,"exact",3.807,917 +"project3602",105,6,4,"none",8.421,917 +"project3602",105,6,4,"union",9.912,917 +"project3602",105,6,4,"exact",9.354,917 +"project3602",105,6,8,"none",19.078,917 +"project3602",105,6,8,"union",21.745,917 +"project3602",105,6,8,"exact",17.409,917 diff --git a/dev/profiling/route3-data/recipe_project3617.csv b/dev/profiling/route3-data/recipe_project3617.csv new file mode 100644 index 000000000..f3c773992 --- /dev/null +++ b/dev/profiling/route3-data/recipe_project3617.csv @@ -0,0 +1,55 @@ +"key","nTip","seed","reps","arm","wall_s","score" +"project3617",65,1,2,"none",3.195,2766 +"project3617",65,1,2,"union",4.846,2762 +"project3617",65,1,2,"exact",4.423,2762 +"project3617",65,1,4,"none",5.71,2762 +"project3617",65,1,4,"union",10.753,2762 +"project3617",65,1,4,"exact",9.546,2762 +"project3617",65,1,8,"none",15.108,2762 +"project3617",65,1,8,"union",18.237,2762 +"project3617",65,1,8,"exact",20.484,2762 +"project3617",65,2,2,"none",3.857,2763 +"project3617",65,2,2,"union",3.771,2769 +"project3617",65,2,2,"exact",5.281,2762 +"project3617",65,2,4,"none",9.21,2763 +"project3617",65,2,4,"union",7.591,2767 +"project3617",65,2,4,"exact",8.326,2762 +"project3617",65,2,8,"none",18.069,2762 +"project3617",65,2,8,"union",18.417,2763 +"project3617",65,2,8,"exact",15.031,2762 +"project3617",65,3,2,"none",3.16,2762 +"project3617",65,3,2,"union",3.651,2767 +"project3617",65,3,2,"exact",4.222,2763 +"project3617",65,3,4,"none",6.408,2762 +"project3617",65,3,4,"union",9.397,2762 +"project3617",65,3,4,"exact",8.922,2763 +"project3617",65,3,8,"none",13.127,2762 +"project3617",65,3,8,"union",16.947,2762 +"project3617",65,3,8,"exact",16.002,2763 +"project3617",65,4,2,"none",3.878,2766 +"project3617",65,4,2,"union",4.698,2763 +"project3617",65,4,2,"exact",3.524,2769 +"project3617",65,4,4,"none",8.834,2762 +"project3617",65,4,4,"union",7.709,2763 +"project3617",65,4,4,"exact",6.661,2763 +"project3617",65,4,8,"none",16.353,2762 +"project3617",65,4,8,"union",15.04,2762 +"project3617",65,4,8,"exact",16.079,2763 +"project3617",65,5,2,"none",3.929,2763 +"project3617",65,5,2,"union",5.311,2762 +"project3617",65,5,2,"exact",4.881,2762 +"project3617",65,5,4,"none",8.537,2762 +"project3617",65,5,4,"union",9.451,2762 +"project3617",65,5,4,"exact",8.859,2762 +"project3617",65,5,8,"none",16.455,2762 +"project3617",65,5,8,"union",18.294,2762 +"project3617",65,5,8,"exact",15.404,2762 +"project3617",65,6,2,"none",3.185,2769 +"project3617",65,6,2,"union",3.299,2770 +"project3617",65,6,2,"exact",3.013,2770 +"project3617",65,6,4,"none",7.127,2767 +"project3617",65,6,4,"union",7.756,2767 +"project3617",65,6,4,"exact",7.724,2767 +"project3617",65,6,8,"none",14.453,2767 +"project3617",65,6,8,"union",18.237,2765 +"project3617",65,6,8,"exact",18.031,2762 diff --git a/dev/profiling/route3-data/recipe_project3807.csv b/dev/profiling/route3-data/recipe_project3807.csv new file mode 100644 index 000000000..f2d10b6fd --- /dev/null +++ b/dev/profiling/route3-data/recipe_project3807.csv @@ -0,0 +1,55 @@ +"key","nTip","seed","reps","arm","wall_s","score" +"project3807",96,1,2,"none",3.469,692 +"project3807",96,1,2,"union",4.702,692 +"project3807",96,1,2,"exact",4.341,692 +"project3807",96,1,4,"none",7.118,692 +"project3807",96,1,4,"union",9.911,692 +"project3807",96,1,4,"exact",8.911,692 +"project3807",96,1,8,"none",16.93,691 +"project3807",96,1,8,"union",22.157,692 +"project3807",96,1,8,"exact",21.157,691 +"project3807",96,2,2,"none",3.256,691 +"project3807",96,2,2,"union",3.651,691 +"project3807",96,2,2,"exact",3.292,691 +"project3807",96,2,4,"none",7.672,691 +"project3807",96,2,4,"union",8.296,691 +"project3807",96,2,4,"exact",7.124,691 +"project3807",96,2,8,"none",15.7,691 +"project3807",96,2,8,"union",19.304,691 +"project3807",96,2,8,"exact",16.781,691 +"project3807",96,3,2,"none",4.43,693 +"project3807",96,3,2,"union",6.658,691 +"project3807",96,3,2,"exact",5.525,692 +"project3807",96,3,4,"none",7.931,692 +"project3807",96,3,4,"union",10.642,691 +"project3807",96,3,4,"exact",11.407,691 +"project3807",96,3,8,"none",17.279,692 +"project3807",96,3,8,"union",18.249,691 +"project3807",96,3,8,"exact",20.043,691 +"project3807",96,4,2,"none",4.725,691 +"project3807",96,4,2,"union",5.123,691 +"project3807",96,4,2,"exact",5.121,691 +"project3807",96,4,4,"none",8.899,691 +"project3807",96,4,4,"union",11.985,691 +"project3807",96,4,4,"exact",11.347,691 +"project3807",96,4,8,"none",17.47,691 +"project3807",96,4,8,"union",23.754,691 +"project3807",96,4,8,"exact",23.613,691 +"project3807",96,5,2,"none",4.707,692 +"project3807",96,5,2,"union",4.713,691 +"project3807",96,5,2,"exact",4.167,691 +"project3807",96,5,4,"none",9.756,691 +"project3807",96,5,4,"union",10.435,691 +"project3807",96,5,4,"exact",8.747,691 +"project3807",96,5,8,"none",18.293,691 +"project3807",96,5,8,"union",24.699,691 +"project3807",96,5,8,"exact",15.612,691 +"project3807",96,6,2,"none",4.925,690 +"project3807",96,6,2,"union",5.852,692 +"project3807",96,6,2,"exact",5.01,691 +"project3807",96,6,4,"none",10.658,690 +"project3807",96,6,4,"union",9.695,692 +"project3807",96,6,4,"exact",10.653,691 +"project3807",96,6,8,"none",19.74,690 +"project3807",96,6,8,"union",19.338,692 +"project3807",96,6,8,"exact",21.903,691 diff --git a/dev/profiling/route3-data/recipe_project4614.csv b/dev/profiling/route3-data/recipe_project4614.csv new file mode 100644 index 000000000..f86732297 --- /dev/null +++ b/dev/profiling/route3-data/recipe_project4614.csv @@ -0,0 +1,55 @@ +"key","nTip","seed","reps","arm","wall_s","score" +"project4614",112,1,2,"none",12.347,1668 +"project4614",112,1,2,"union",13.462,1668 +"project4614",112,1,2,"exact",10.811,1668 +"project4614",112,1,4,"none",24.019,1665 +"project4614",112,1,4,"union",27.672,1666 +"project4614",112,1,4,"exact",22.605,1665 +"project4614",112,1,8,"none",44.488,1663 +"project4614",112,1,8,"union",53.121,1666 +"project4614",112,1,8,"exact",44.275,1664 +"project4614",112,2,2,"none",8.084,1665 +"project4614",112,2,2,"union",12.424,1665 +"project4614",112,2,2,"exact",9.079,1665 +"project4614",112,2,4,"none",21.482,1665 +"project4614",112,2,4,"union",22.332,1665 +"project4614",112,2,4,"exact",17.774,1665 +"project4614",112,2,8,"none",41.125,1662 +"project4614",112,2,8,"union",48.503,1664 +"project4614",112,2,8,"exact",37.348,1665 +"project4614",112,3,2,"none",8.845,1666 +"project4614",112,3,2,"union",11.009,1663 +"project4614",112,3,2,"exact",9.312,1663 +"project4614",112,3,4,"none",21.807,1665 +"project4614",112,3,4,"union",21.159,1663 +"project4614",112,3,4,"exact",18.233,1664 +"project4614",112,3,8,"none",45.094,1665 +"project4614",112,3,8,"union",44.345,1663 +"project4614",112,3,8,"exact",37.586,1664 +"project4614",112,4,2,"none",8.985,1671 +"project4614",112,4,2,"union",11.387,1670 +"project4614",112,4,2,"exact",12.064,1667 +"project4614",112,4,4,"none",15.501,1669 +"project4614",112,4,4,"union",21.814,1661 +"project4614",112,4,4,"exact",23.999,1664 +"project4614",112,4,8,"none",35.812,1668 +"project4614",112,4,8,"union",43.058,1661 +"project4614",112,4,8,"exact",48.136,1664 +"project4614",112,5,2,"none",7.973,1668 +"project4614",112,5,2,"union",10.63,1663 +"project4614",112,5,2,"exact",10.961,1672 +"project4614",112,5,4,"none",18.741,1663 +"project4614",112,5,4,"union",18,1663 +"project4614",112,5,4,"exact",18.223,1672 +"project4614",112,5,8,"none",37.733,1663 +"project4614",112,5,8,"union",34.632,1663 +"project4614",112,5,8,"exact",36.62,1662 +"project4614",112,6,2,"none",8.514,1663 +"project4614",112,6,2,"union",13.816,1667 +"project4614",112,6,2,"exact",8.686,1665 +"project4614",112,6,4,"none",17.369,1663 +"project4614",112,6,4,"union",25.603,1665 +"project4614",112,6,4,"exact",19.075,1665 +"project4614",112,6,8,"none",35.206,1663 +"project4614",112,6,8,"union",50.351,1662 +"project4614",112,6,8,"exact",39.029,1663 diff --git a/dev/profiling/route3-data/recipe_project4624.csv b/dev/profiling/route3-data/recipe_project4624.csv new file mode 100644 index 000000000..70f68024d --- /dev/null +++ b/dev/profiling/route3-data/recipe_project4624.csv @@ -0,0 +1,55 @@ +"key","nTip","seed","reps","arm","wall_s","score" +"project4624",76,1,2,"none",3.676,2196 +"project4624",76,1,2,"union",4.59,2196 +"project4624",76,1,2,"exact",8.277,2196 +"project4624",76,1,4,"none",19.444,2196 +"project4624",76,1,4,"union",20.528,2196 +"project4624",76,1,4,"exact",19.118,2196 +"project4624",76,1,8,"none",40.613,2196 +"project4624",76,1,8,"union",49.441,2196 +"project4624",76,1,8,"exact",45.341,2196 +"project4624",76,2,2,"none",10.112,2196 +"project4624",76,2,2,"union",12.795,2196 +"project4624",76,2,2,"exact",11.823,2196 +"project4624",76,2,4,"none",19.619,2196 +"project4624",76,2,4,"union",12.278,2196 +"project4624",76,2,4,"exact",11.443,2196 +"project4624",76,2,8,"none",36.161,2196 +"project4624",76,2,8,"union",55.85,2196 +"project4624",76,2,8,"exact",51.413,2196 +"project4624",76,3,2,"none",9.959,2196 +"project4624",76,3,2,"union",5.43,2196 +"project4624",76,3,2,"exact",4.36,2196 +"project4624",76,3,4,"none",21.16,2196 +"project4624",76,3,4,"union",14.653,2196 +"project4624",76,3,4,"exact",8.696,2196 +"project4624",76,3,8,"none",18.128,2196 +"project4624",76,3,8,"union",43.15,2196 +"project4624",76,3,8,"exact",47.235,2196 +"project4624",76,4,2,"none",15.485,2196 +"project4624",76,4,2,"union",12.374,2196 +"project4624",76,4,2,"exact",8.953,2196 +"project4624",76,4,4,"none",25.507,2196 +"project4624",76,4,4,"union",33.388,2196 +"project4624",76,4,4,"exact",22.897,2196 +"project4624",76,4,8,"none",46.042,2196 +"project4624",76,4,8,"union",53.93,2196 +"project4624",76,4,8,"exact",46.686,2196 +"project4624",76,5,2,"none",11.125,2196 +"project4624",76,5,2,"union",11.973,2196 +"project4624",76,5,2,"exact",10.813,2196 +"project4624",76,5,4,"none",16.724,2196 +"project4624",76,5,4,"union",20.655,2196 +"project4624",76,5,4,"exact",21.564,2196 +"project4624",76,5,8,"none",36.018,2196 +"project4624",76,5,8,"union",42.855,2196 +"project4624",76,5,8,"exact",41.207,2196 +"project4624",76,6,2,"none",7.536,2196 +"project4624",76,6,2,"union",10.716,2196 +"project4624",76,6,2,"exact",9.882,2196 +"project4624",76,6,4,"none",20.524,2196 +"project4624",76,6,4,"union",23.385,2196 +"project4624",76,6,4,"exact",21.324,2196 +"project4624",76,6,8,"none",35.188,2196 +"project4624",76,6,8,"union",47.612,2196 +"project4624",76,6,8,"exact",44.937,2196 diff --git a/dev/profiling/route3-data/recipe_project5201.csv b/dev/profiling/route3-data/recipe_project5201.csv new file mode 100644 index 000000000..7226352c8 --- /dev/null +++ b/dev/profiling/route3-data/recipe_project5201.csv @@ -0,0 +1,55 @@ +"key","nTip","seed","reps","arm","wall_s","score" +"project5201",86,1,2,"none",1.799,233 +"project5201",86,1,2,"union",2.369,234 +"project5201",86,1,2,"exact",2.057,233 +"project5201",86,1,4,"none",3.337,233 +"project5201",86,1,4,"union",4.054,233 +"project5201",86,1,4,"exact",3.754,233 +"project5201",86,1,8,"none",6.075,233 +"project5201",86,1,8,"union",7.254,233 +"project5201",86,1,8,"exact",6.464,233 +"project5201",86,2,2,"none",1.489,234 +"project5201",86,2,2,"union",2.052,235 +"project5201",86,2,2,"exact",1.576,234 +"project5201",86,2,4,"none",2.321,234 +"project5201",86,2,4,"union",3.734,234 +"project5201",86,2,4,"exact",2.893,234 +"project5201",86,2,8,"none",4.796,233 +"project5201",86,2,8,"union",6.429,233 +"project5201",86,2,8,"exact",6.129,233 +"project5201",86,3,2,"none",1.491,233 +"project5201",86,3,2,"union",1.655,233 +"project5201",86,3,2,"exact",1.517,233 +"project5201",86,3,4,"none",2.839,233 +"project5201",86,3,4,"union",3.249,233 +"project5201",86,3,4,"exact",3.015,233 +"project5201",86,3,8,"none",5.869,233 +"project5201",86,3,8,"union",6.253,233 +"project5201",86,3,8,"exact",5.235,233 +"project5201",86,4,2,"none",1.019,235 +"project5201",86,4,2,"union",1.217,234 +"project5201",86,4,2,"exact",1.287,234 +"project5201",86,4,4,"none",3.083,233 +"project5201",86,4,4,"union",3.26,234 +"project5201",86,4,4,"exact",2.401,233 +"project5201",86,4,8,"none",5.906,233 +"project5201",86,4,8,"union",5.576,233 +"project5201",86,4,8,"exact",5.077,233 +"project5201",86,5,2,"none",0.97,233 +"project5201",86,5,2,"union",2.059,233 +"project5201",86,5,2,"exact",1.88,233 +"project5201",86,5,4,"none",3.335,233 +"project5201",86,5,4,"union",3.942,233 +"project5201",86,5,4,"exact",3.115,233 +"project5201",86,5,8,"none",6.547,233 +"project5201",86,5,8,"union",6.799,233 +"project5201",86,5,8,"exact",5.997,233 +"project5201",86,6,2,"none",0.888,233 +"project5201",86,6,2,"union",1.375,234 +"project5201",86,6,2,"exact",1.239,234 +"project5201",86,6,4,"none",2.42,233 +"project5201",86,6,4,"union",2.938,233 +"project5201",86,6,4,"exact",2.583,234 +"project5201",86,6,8,"none",5.322,233 +"project5201",86,6,8,"union",5.392,233 +"project5201",86,6,8,"exact",5.578,233 diff --git a/dev/profiling/route3-data/recipe_project589.csv b/dev/profiling/route3-data/recipe_project589.csv new file mode 100644 index 000000000..1f54774f5 --- /dev/null +++ b/dev/profiling/route3-data/recipe_project589.csv @@ -0,0 +1,55 @@ +"key","nTip","seed","reps","arm","wall_s","score" +"project589",69,1,2,"none",1.201,368 +"project589",69,1,2,"union",1.551,368 +"project589",69,1,2,"exact",1.613,368 +"project589",69,1,4,"none",2.632,368 +"project589",69,1,4,"union",3.097,368 +"project589",69,1,4,"exact",2.961,368 +"project589",69,1,8,"none",5.782,368 +"project589",69,1,8,"union",6.438,368 +"project589",69,1,8,"exact",5.358,368 +"project589",69,2,2,"none",1.027,368 +"project589",69,2,2,"union",1.378,368 +"project589",69,2,2,"exact",1.36,368 +"project589",69,2,4,"none",2.232,368 +"project589",69,2,4,"union",2.723,368 +"project589",69,2,4,"exact",2.689,368 +"project589",69,2,8,"none",4.776,368 +"project589",69,2,8,"union",5.424,368 +"project589",69,2,8,"exact",5.374,368 +"project589",69,3,2,"none",1.456,368 +"project589",69,3,2,"union",1.29,368 +"project589",69,3,2,"exact",1.287,368 +"project589",69,3,4,"none",2.351,368 +"project589",69,3,4,"union",2.172,368 +"project589",69,3,4,"exact",2.128,368 +"project589",69,3,8,"none",4.949,368 +"project589",69,3,8,"union",4.571,368 +"project589",69,3,8,"exact",4.497,368 +"project589",69,4,2,"none",1.206,368 +"project589",69,4,2,"union",1.215,368 +"project589",69,4,2,"exact",1.215,368 +"project589",69,4,4,"none",2.696,368 +"project589",69,4,4,"union",2.694,368 +"project589",69,4,4,"exact",2.677,368 +"project589",69,4,8,"none",5.075,368 +"project589",69,4,8,"union",5.424,368 +"project589",69,4,8,"exact",5.379,368 +"project589",69,5,2,"none",1.902,368 +"project589",69,5,2,"union",1.572,368 +"project589",69,5,2,"exact",1.556,368 +"project589",69,5,4,"none",3.111,368 +"project589",69,5,4,"union",2.922,368 +"project589",69,5,4,"exact",2.852,368 +"project589",69,5,8,"none",6.018,368 +"project589",69,5,8,"union",5.26,368 +"project589",69,5,8,"exact",5.182,368 +"project589",69,6,2,"none",1.601,368 +"project589",69,6,2,"union",1.853,368 +"project589",69,6,2,"exact",1.832,368 +"project589",69,6,4,"none",2.721,368 +"project589",69,6,4,"union",3.035,368 +"project589",69,6,4,"exact",2.997,368 +"project589",69,6,8,"none",5.503,368 +"project589",69,6,8,"union",6.296,368 +"project589",69,6,8,"exact",6.146,368 diff --git a/dev/profiling/route3-data/repl_Dikow2009.csv b/dev/profiling/route3-data/repl_Dikow2009.csv new file mode 100644 index 000000000..ea3e151cd --- /dev/null +++ b/dev/profiling/route3-data/repl_Dikow2009.csv @@ -0,0 +1,101 @@ +"dataset","nTip","seed","cycles","arm","wall_s","score" +"Dikow2009",88,1,1,"replicate",0.346,1611 +"Dikow2009",88,1,2,"replicate",0.716,1610 +"Dikow2009",88,1,4,"replicate",1.343,1608 +"Dikow2009",88,1,8,"replicate",2.594,1608 +"Dikow2009",88,1,16,"replicate",4.945,1608 +"Dikow2009",88,2,1,"replicate",0.332,1608 +"Dikow2009",88,2,2,"replicate",0.637,1608 +"Dikow2009",88,2,4,"replicate",1.233,1608 +"Dikow2009",88,2,8,"replicate",2.489,1608 +"Dikow2009",88,2,16,"replicate",4.971,1608 +"Dikow2009",88,3,1,"replicate",0.367,1614 +"Dikow2009",88,3,2,"replicate",0.725,1609 +"Dikow2009",88,3,4,"replicate",1.345,1609 +"Dikow2009",88,3,8,"replicate",2.531,1609 +"Dikow2009",88,3,16,"replicate",5.09,1608 +"Dikow2009",88,4,1,"replicate",0.308,1610 +"Dikow2009",88,4,2,"replicate",0.647,1609 +"Dikow2009",88,4,4,"replicate",1.312,1609 +"Dikow2009",88,4,8,"replicate",2.453,1609 +"Dikow2009",88,4,16,"replicate",5.192,1609 +"Dikow2009",88,5,1,"replicate",0.339,1608 +"Dikow2009",88,5,2,"replicate",0.605,1608 +"Dikow2009",88,5,4,"replicate",1.339,1608 +"Dikow2009",88,5,8,"replicate",2.624,1608 +"Dikow2009",88,5,16,"replicate",5.218,1608 +"Dikow2009",88,6,1,"replicate",0.29,1609 +"Dikow2009",88,6,2,"replicate",0.616,1609 +"Dikow2009",88,6,4,"replicate",1.165,1609 +"Dikow2009",88,6,8,"replicate",2.486,1609 +"Dikow2009",88,6,16,"replicate",4.899,1608 +"Dikow2009",88,7,1,"replicate",0.315,1611 +"Dikow2009",88,7,2,"replicate",0.599,1609 +"Dikow2009",88,7,4,"replicate",1.235,1609 +"Dikow2009",88,7,8,"replicate",2.584,1609 +"Dikow2009",88,7,16,"replicate",5.055,1609 +"Dikow2009",88,8,1,"replicate",0.368,1607 +"Dikow2009",88,8,2,"replicate",0.684,1607 +"Dikow2009",88,8,4,"replicate",1.357,1607 +"Dikow2009",88,8,8,"replicate",2.628,1607 +"Dikow2009",88,8,16,"replicate",5.155,1607 +"Dikow2009",88,9,1,"replicate",0.373,1613 +"Dikow2009",88,9,2,"replicate",0.748,1610 +"Dikow2009",88,9,4,"replicate",1.338,1609 +"Dikow2009",88,9,8,"replicate",2.583,1608 +"Dikow2009",88,9,16,"replicate",5.229,1608 +"Dikow2009",88,10,1,"replicate",0.291,1614 +"Dikow2009",88,10,2,"replicate",0.565,1614 +"Dikow2009",88,10,4,"replicate",1.114,1610 +"Dikow2009",88,10,8,"replicate",2.379,1609 +"Dikow2009",88,10,16,"replicate",4.837,1608 +"Dikow2009",88,11,1,"replicate",0.318,1607 +"Dikow2009",88,11,2,"replicate",0.626,1607 +"Dikow2009",88,11,4,"replicate",1.206,1607 +"Dikow2009",88,11,8,"replicate",2.413,1607 +"Dikow2009",88,11,16,"replicate",4.893,1607 +"Dikow2009",88,12,1,"replicate",0.343,1612 +"Dikow2009",88,12,2,"replicate",0.618,1612 +"Dikow2009",88,12,4,"replicate",1.233,1610 +"Dikow2009",88,12,8,"replicate",2.606,1608 +"Dikow2009",88,12,16,"replicate",4.936,1607 +"Dikow2009",88,13,1,"replicate",0.302,1608 +"Dikow2009",88,13,2,"replicate",0.593,1608 +"Dikow2009",88,13,4,"replicate",1.185,1608 +"Dikow2009",88,13,8,"replicate",2.447,1608 +"Dikow2009",88,13,16,"replicate",5.123,1608 +"Dikow2009",88,14,1,"replicate",0.388,1610 +"Dikow2009",88,14,2,"replicate",0.691,1607 +"Dikow2009",88,14,4,"replicate",1.283,1607 +"Dikow2009",88,14,8,"replicate",2.631,1607 +"Dikow2009",88,14,16,"replicate",5.233,1607 +"Dikow2009",88,15,1,"replicate",0.324,1610 +"Dikow2009",88,15,2,"replicate",0.675,1608 +"Dikow2009",88,15,4,"replicate",1.365,1608 +"Dikow2009",88,15,8,"replicate",2.547,1608 +"Dikow2009",88,15,16,"replicate",5.132,1607 +"Dikow2009",88,16,1,"replicate",0.303,1608 +"Dikow2009",88,16,2,"replicate",0.617,1608 +"Dikow2009",88,16,4,"replicate",1.194,1608 +"Dikow2009",88,16,8,"replicate",2.611,1608 +"Dikow2009",88,16,16,"replicate",5.088,1608 +"Dikow2009",88,17,1,"replicate",0.244,1611 +"Dikow2009",88,17,2,"replicate",0.576,1608 +"Dikow2009",88,17,4,"replicate",1.237,1608 +"Dikow2009",88,17,8,"replicate",2.527,1607 +"Dikow2009",88,17,16,"replicate",5.109,1607 +"Dikow2009",88,18,1,"replicate",0.252,1609 +"Dikow2009",88,18,2,"replicate",0.584,1609 +"Dikow2009",88,18,4,"replicate",1.324,1609 +"Dikow2009",88,18,8,"replicate",2.583,1609 +"Dikow2009",88,18,16,"replicate",5.28,1607 +"Dikow2009",88,19,1,"replicate",0.289,1608 +"Dikow2009",88,19,2,"replicate",0.633,1608 +"Dikow2009",88,19,4,"replicate",1.302,1608 +"Dikow2009",88,19,8,"replicate",2.549,1608 +"Dikow2009",88,19,16,"replicate",5.183,1607 +"Dikow2009",88,20,1,"replicate",0.331,1612 +"Dikow2009",88,20,2,"replicate",0.634,1612 +"Dikow2009",88,20,4,"replicate",1.306,1608 +"Dikow2009",88,20,8,"replicate",2.633,1607 +"Dikow2009",88,20,16,"replicate",5.301,1607 diff --git a/dev/profiling/route3-data/repl_Zanol2014.csv b/dev/profiling/route3-data/repl_Zanol2014.csv new file mode 100644 index 000000000..e3dbb3ccb --- /dev/null +++ b/dev/profiling/route3-data/repl_Zanol2014.csv @@ -0,0 +1,101 @@ +"dataset","nTip","seed","cycles","arm","wall_s","score" +"Zanol2014",74,1,1,"replicate",0.144,1272 +"Zanol2014",74,1,2,"replicate",0.324,1265 +"Zanol2014",74,1,4,"replicate",0.69,1265 +"Zanol2014",74,1,8,"replicate",1.371,1265 +"Zanol2014",74,1,16,"replicate",2.741,1263 +"Zanol2014",74,2,1,"replicate",0.184,1269 +"Zanol2014",74,2,2,"replicate",0.358,1269 +"Zanol2014",74,2,4,"replicate",0.701,1266 +"Zanol2014",74,2,8,"replicate",1.307,1262 +"Zanol2014",74,2,16,"replicate",2.609,1262 +"Zanol2014",74,3,1,"replicate",0.203,1263 +"Zanol2014",74,3,2,"replicate",0.37,1263 +"Zanol2014",74,3,4,"replicate",0.668,1263 +"Zanol2014",74,3,8,"replicate",1.36,1263 +"Zanol2014",74,3,16,"replicate",2.67,1262 +"Zanol2014",74,4,1,"replicate",0.151,1265 +"Zanol2014",74,4,2,"replicate",0.32,1264 +"Zanol2014",74,4,4,"replicate",0.68,1264 +"Zanol2014",74,4,8,"replicate",1.323,1264 +"Zanol2014",74,4,16,"replicate",2.658,1264 +"Zanol2014",74,5,1,"replicate",0.142,1262 +"Zanol2014",74,5,2,"replicate",0.336,1262 +"Zanol2014",74,5,4,"replicate",0.642,1262 +"Zanol2014",74,5,8,"replicate",1.26,1262 +"Zanol2014",74,5,16,"replicate",2.538,1262 +"Zanol2014",74,6,1,"replicate",0.165,1268 +"Zanol2014",74,6,2,"replicate",0.344,1268 +"Zanol2014",74,6,4,"replicate",0.71,1264 +"Zanol2014",74,6,8,"replicate",1.326,1264 +"Zanol2014",74,6,16,"replicate",2.621,1264 +"Zanol2014",74,7,1,"replicate",0.183,1267 +"Zanol2014",74,7,2,"replicate",0.358,1266 +"Zanol2014",74,7,4,"replicate",0.699,1263 +"Zanol2014",74,7,8,"replicate",1.394,1263 +"Zanol2014",74,7,16,"replicate",2.669,1263 +"Zanol2014",74,8,1,"replicate",0.17,1265 +"Zanol2014",74,8,2,"replicate",0.326,1265 +"Zanol2014",74,8,4,"replicate",0.651,1265 +"Zanol2014",74,8,8,"replicate",1.278,1263 +"Zanol2014",74,8,16,"replicate",2.588,1263 +"Zanol2014",74,9,1,"replicate",0.138,1273 +"Zanol2014",74,9,2,"replicate",0.287,1273 +"Zanol2014",74,9,4,"replicate",0.61,1264 +"Zanol2014",74,9,8,"replicate",1.314,1262 +"Zanol2014",74,9,16,"replicate",2.718,1262 +"Zanol2014",74,10,1,"replicate",0.224,1264 +"Zanol2014",74,10,2,"replicate",0.38,1264 +"Zanol2014",74,10,4,"replicate",0.701,1263 +"Zanol2014",74,10,8,"replicate",1.288,1263 +"Zanol2014",74,10,16,"replicate",2.61,1263 +"Zanol2014",74,11,1,"replicate",0.172,1268 +"Zanol2014",74,11,2,"replicate",0.314,1268 +"Zanol2014",74,11,4,"replicate",0.618,1264 +"Zanol2014",74,11,8,"replicate",1.263,1262 +"Zanol2014",74,11,16,"replicate",2.616,1262 +"Zanol2014",74,12,1,"replicate",0.183,1264 +"Zanol2014",74,12,2,"replicate",0.326,1264 +"Zanol2014",74,12,4,"replicate",0.623,1264 +"Zanol2014",74,12,8,"replicate",1.283,1263 +"Zanol2014",74,12,16,"replicate",2.683,1263 +"Zanol2014",74,13,1,"replicate",0.154,1266 +"Zanol2014",74,13,2,"replicate",0.357,1266 +"Zanol2014",74,13,4,"replicate",0.656,1266 +"Zanol2014",74,13,8,"replicate",1.346,1264 +"Zanol2014",74,13,16,"replicate",2.713,1264 +"Zanol2014",74,14,1,"replicate",0.18,1265 +"Zanol2014",74,14,2,"replicate",0.375,1265 +"Zanol2014",74,14,4,"replicate",0.72,1265 +"Zanol2014",74,14,8,"replicate",1.284,1264 +"Zanol2014",74,14,16,"replicate",2.541,1263 +"Zanol2014",74,15,1,"replicate",0.175,1263 +"Zanol2014",74,15,2,"replicate",0.341,1263 +"Zanol2014",74,15,4,"replicate",0.716,1263 +"Zanol2014",74,15,8,"replicate",1.353,1263 +"Zanol2014",74,15,16,"replicate",2.683,1263 +"Zanol2014",74,16,1,"replicate",0.157,1269 +"Zanol2014",74,16,2,"replicate",0.297,1269 +"Zanol2014",74,16,4,"replicate",0.618,1264 +"Zanol2014",74,16,8,"replicate",1.185,1264 +"Zanol2014",74,16,16,"replicate",2.49,1264 +"Zanol2014",74,17,1,"replicate",0.196,1266 +"Zanol2014",74,17,2,"replicate",0.365,1262 +"Zanol2014",74,17,4,"replicate",0.736,1262 +"Zanol2014",74,17,8,"replicate",1.465,1262 +"Zanol2014",74,17,16,"replicate",2.703,1262 +"Zanol2014",74,18,1,"replicate",0.169,1264 +"Zanol2014",74,18,2,"replicate",0.362,1264 +"Zanol2014",74,18,4,"replicate",0.66,1263 +"Zanol2014",74,18,8,"replicate",1.346,1263 +"Zanol2014",74,18,16,"replicate",2.736,1263 +"Zanol2014",74,19,1,"replicate",0.159,1267 +"Zanol2014",74,19,2,"replicate",0.337,1267 +"Zanol2014",74,19,4,"replicate",0.691,1267 +"Zanol2014",74,19,8,"replicate",1.317,1266 +"Zanol2014",74,19,16,"replicate",2.663,1262 +"Zanol2014",74,20,1,"replicate",0.148,1266 +"Zanol2014",74,20,2,"replicate",0.303,1266 +"Zanol2014",74,20,4,"replicate",0.651,1266 +"Zanol2014",74,20,8,"replicate",1.374,1262 +"Zanol2014",74,20,16,"replicate",2.727,1262 diff --git a/dev/profiling/route3-data/repl_Zhu2013.csv b/dev/profiling/route3-data/repl_Zhu2013.csv new file mode 100644 index 000000000..94ef02006 --- /dev/null +++ b/dev/profiling/route3-data/repl_Zhu2013.csv @@ -0,0 +1,101 @@ +"dataset","nTip","seed","cycles","arm","wall_s","score" +"Zhu2013",75,1,1,"replicate",0.117,631 +"Zhu2013",75,1,2,"replicate",0.218,629 +"Zhu2013",75,1,4,"replicate",0.483,628 +"Zhu2013",75,1,8,"replicate",1.015,628 +"Zhu2013",75,1,16,"replicate",2.103,625 +"Zhu2013",75,2,1,"replicate",0.119,630 +"Zhu2013",75,2,2,"replicate",0.227,630 +"Zhu2013",75,2,4,"replicate",0.483,627 +"Zhu2013",75,2,8,"replicate",0.927,627 +"Zhu2013",75,2,16,"replicate",2.64,625 +"Zhu2013",75,3,1,"replicate",0.251,626 +"Zhu2013",75,3,2,"replicate",0.492,626 +"Zhu2013",75,3,4,"replicate",1.002,626 +"Zhu2013",75,3,8,"replicate",2.006,626 +"Zhu2013",75,3,16,"replicate",4.124,625 +"Zhu2013",75,4,1,"replicate",0.276,627 +"Zhu2013",75,4,2,"replicate",0.56,627 +"Zhu2013",75,4,4,"replicate",1.061,627 +"Zhu2013",75,4,8,"replicate",2.012,625 +"Zhu2013",75,4,16,"replicate",3.903,625 +"Zhu2013",75,5,1,"replicate",0.155,626 +"Zhu2013",75,5,2,"replicate",0.374,626 +"Zhu2013",75,5,4,"replicate",0.886,625 +"Zhu2013",75,5,8,"replicate",1.918,625 +"Zhu2013",75,5,16,"replicate",3.782,625 +"Zhu2013",75,6,1,"replicate",0.279,626 +"Zhu2013",75,6,2,"replicate",0.452,626 +"Zhu2013",75,6,4,"replicate",0.988,626 +"Zhu2013",75,6,8,"replicate",1.948,624 +"Zhu2013",75,6,16,"replicate",3.935,624 +"Zhu2013",75,7,1,"replicate",0.254,626 +"Zhu2013",75,7,2,"replicate",0.506,626 +"Zhu2013",75,7,4,"replicate",0.977,626 +"Zhu2013",75,7,8,"replicate",2.082,626 +"Zhu2013",75,7,16,"replicate",4.091,625 +"Zhu2013",75,8,1,"replicate",0.264,628 +"Zhu2013",75,8,2,"replicate",0.533,628 +"Zhu2013",75,8,4,"replicate",1.055,628 +"Zhu2013",75,8,8,"replicate",2.199,628 +"Zhu2013",75,8,16,"replicate",3.95,625 +"Zhu2013",75,9,1,"replicate",0.111,629 +"Zhu2013",75,9,2,"replicate",0.205,629 +"Zhu2013",75,9,4,"replicate",0.447,627 +"Zhu2013",75,9,8,"replicate",0.925,627 +"Zhu2013",75,9,16,"replicate",1.841,626 +"Zhu2013",75,10,1,"replicate",0.112,627 +"Zhu2013",75,10,2,"replicate",0.221,627 +"Zhu2013",75,10,4,"replicate",0.475,627 +"Zhu2013",75,10,8,"replicate",0.936,627 +"Zhu2013",75,10,16,"replicate",1.875,625 +"Zhu2013",75,11,1,"replicate",0.13,629 +"Zhu2013",75,11,2,"replicate",0.246,629 +"Zhu2013",75,11,4,"replicate",0.471,629 +"Zhu2013",75,11,8,"replicate",0.9,628 +"Zhu2013",75,11,16,"replicate",1.799,626 +"Zhu2013",75,12,1,"replicate",0.111,627 +"Zhu2013",75,12,2,"replicate",0.218,627 +"Zhu2013",75,12,4,"replicate",0.477,627 +"Zhu2013",75,12,8,"replicate",0.921,627 +"Zhu2013",75,12,16,"replicate",1.888,626 +"Zhu2013",75,13,1,"replicate",0.091,625 +"Zhu2013",75,13,2,"replicate",0.212,625 +"Zhu2013",75,13,4,"replicate",0.448,625 +"Zhu2013",75,13,8,"replicate",0.936,625 +"Zhu2013",75,13,16,"replicate",1.833,624 +"Zhu2013",75,14,1,"replicate",0.106,627 +"Zhu2013",75,14,2,"replicate",0.225,627 +"Zhu2013",75,14,4,"replicate",0.414,625 +"Zhu2013",75,14,8,"replicate",0.852,625 +"Zhu2013",75,14,16,"replicate",1.65,625 +"Zhu2013",75,15,1,"replicate",0.135,633 +"Zhu2013",75,15,2,"replicate",0.263,630 +"Zhu2013",75,15,4,"replicate",0.448,628 +"Zhu2013",75,15,8,"replicate",0.947,627 +"Zhu2013",75,15,16,"replicate",1.82,626 +"Zhu2013",75,16,1,"replicate",0.126,629 +"Zhu2013",75,16,2,"replicate",0.242,626 +"Zhu2013",75,16,4,"replicate",0.457,626 +"Zhu2013",75,16,8,"replicate",0.924,626 +"Zhu2013",75,16,16,"replicate",1.851,625 +"Zhu2013",75,17,1,"replicate",0.111,629 +"Zhu2013",75,17,2,"replicate",0.205,628 +"Zhu2013",75,17,4,"replicate",0.454,628 +"Zhu2013",75,17,8,"replicate",0.934,626 +"Zhu2013",75,17,16,"replicate",1.838,626 +"Zhu2013",75,18,1,"replicate",0.106,630 +"Zhu2013",75,18,2,"replicate",0.229,627 +"Zhu2013",75,18,4,"replicate",0.413,626 +"Zhu2013",75,18,8,"replicate",0.874,626 +"Zhu2013",75,18,16,"replicate",1.823,626 +"Zhu2013",75,19,1,"replicate",0.114,627 +"Zhu2013",75,19,2,"replicate",0.233,627 +"Zhu2013",75,19,4,"replicate",0.471,627 +"Zhu2013",75,19,8,"replicate",0.929,627 +"Zhu2013",75,19,16,"replicate",1.794,626 +"Zhu2013",75,20,1,"replicate",0.105,628 +"Zhu2013",75,20,2,"replicate",0.221,628 +"Zhu2013",75,20,4,"replicate",0.433,627 +"Zhu2013",75,20,8,"replicate",0.949,627 +"Zhu2013",75,20,16,"replicate",1.887,627 diff --git a/dev/profiling/route3-data/route3_Dikow2009.csv b/dev/profiling/route3-data/route3_Dikow2009.csv new file mode 100644 index 000000000..443303147 --- /dev/null +++ b/dev/profiling/route3-data/route3_Dikow2009.csv @@ -0,0 +1,301 @@ +"dataset","nTip","seed","cycles","arm","wall_s","score" +"Dikow2009",88,1,1,"ratchet",0.068,1611 +"Dikow2009",88,1,1,"drift_exact",0.031,1610 +"Dikow2009",88,1,1,"drift_union",0.045,1608 +"Dikow2009",88,1,2,"ratchet",0.124,1611 +"Dikow2009",88,1,2,"drift_exact",0.056,1610 +"Dikow2009",88,1,2,"drift_union",0.072,1608 +"Dikow2009",88,1,4,"ratchet",0.22,1608 +"Dikow2009",88,1,4,"drift_exact",0.099,1610 +"Dikow2009",88,1,4,"drift_union",0.116,1608 +"Dikow2009",88,1,8,"ratchet",0.311,1608 +"Dikow2009",88,1,8,"drift_exact",0.173,1609 +"Dikow2009",88,1,8,"drift_union",0.196,1607 +"Dikow2009",88,1,16,"ratchet",0.57,1608 +"Dikow2009",88,1,16,"drift_exact",0.347,1608 +"Dikow2009",88,1,16,"drift_union",0.407,1607 +"Dikow2009",88,2,1,"ratchet",0.041,1610 +"Dikow2009",88,2,1,"drift_exact",0.031,1609 +"Dikow2009",88,2,1,"drift_union",0.038,1609 +"Dikow2009",88,2,2,"ratchet",0.078,1610 +"Dikow2009",88,2,2,"drift_exact",0.046,1608 +"Dikow2009",88,2,2,"drift_union",0.06,1609 +"Dikow2009",88,2,4,"ratchet",0.151,1610 +"Dikow2009",88,2,4,"drift_exact",0.085,1608 +"Dikow2009",88,2,4,"drift_union",0.11,1609 +"Dikow2009",88,2,8,"ratchet",0.264,1609 +"Dikow2009",88,2,8,"drift_exact",0.184,1608 +"Dikow2009",88,2,8,"drift_union",0.263,1608 +"Dikow2009",88,2,16,"ratchet",0.469,1609 +"Dikow2009",88,2,16,"drift_exact",0.387,1608 +"Dikow2009",88,2,16,"drift_union",0.459,1608 +"Dikow2009",88,3,1,"ratchet",0.066,1611 +"Dikow2009",88,3,1,"drift_exact",0.047,1609 +"Dikow2009",88,3,1,"drift_union",0.03,1611 +"Dikow2009",88,3,2,"ratchet",0.092,1611 +"Dikow2009",88,3,2,"drift_exact",0.059,1609 +"Dikow2009",88,3,2,"drift_union",0.042,1611 +"Dikow2009",88,3,4,"ratchet",0.139,1610 +"Dikow2009",88,3,4,"drift_exact",0.111,1607 +"Dikow2009",88,3,4,"drift_union",0.088,1611 +"Dikow2009",88,3,8,"ratchet",0.271,1609 +"Dikow2009",88,3,8,"drift_exact",0.196,1607 +"Dikow2009",88,3,8,"drift_union",0.182,1610 +"Dikow2009",88,3,16,"ratchet",0.5,1608 +"Dikow2009",88,3,16,"drift_exact",0.392,1606 +"Dikow2009",88,3,16,"drift_union",0.389,1608 +"Dikow2009",88,4,1,"ratchet",0.046,1610 +"Dikow2009",88,4,1,"drift_exact",0.034,1610 +"Dikow2009",88,4,1,"drift_union",0.059,1610 +"Dikow2009",88,4,2,"ratchet",0.074,1610 +"Dikow2009",88,4,2,"drift_exact",0.054,1610 +"Dikow2009",88,4,2,"drift_union",0.072,1610 +"Dikow2009",88,4,4,"ratchet",0.123,1610 +"Dikow2009",88,4,4,"drift_exact",0.109,1610 +"Dikow2009",88,4,4,"drift_union",0.123,1609 +"Dikow2009",88,4,8,"ratchet",0.222,1609 +"Dikow2009",88,4,8,"drift_exact",0.205,1610 +"Dikow2009",88,4,8,"drift_union",0.202,1609 +"Dikow2009",88,4,16,"ratchet",0.402,1609 +"Dikow2009",88,4,16,"drift_exact",0.362,1610 +"Dikow2009",88,4,16,"drift_union",0.388,1609 +"Dikow2009",88,5,1,"ratchet",0.035,1608 +"Dikow2009",88,5,1,"drift_exact",0.04,1607 +"Dikow2009",88,5,1,"drift_union",0.027,1608 +"Dikow2009",88,5,2,"ratchet",0.057,1608 +"Dikow2009",88,5,2,"drift_exact",0.057,1607 +"Dikow2009",88,5,2,"drift_union",0.045,1608 +"Dikow2009",88,5,4,"ratchet",0.102,1608 +"Dikow2009",88,5,4,"drift_exact",0.104,1607 +"Dikow2009",88,5,4,"drift_union",0.094,1607 +"Dikow2009",88,5,8,"ratchet",0.208,1607 +"Dikow2009",88,5,8,"drift_exact",0.214,1607 +"Dikow2009",88,5,8,"drift_union",0.203,1607 +"Dikow2009",88,5,16,"ratchet",0.417,1607 +"Dikow2009",88,5,16,"drift_exact",0.383,1607 +"Dikow2009",88,5,16,"drift_union",0.417,1607 +"Dikow2009",88,6,1,"ratchet",0.05,1608 +"Dikow2009",88,6,1,"drift_exact",0.033,1608 +"Dikow2009",88,6,1,"drift_union",0.025,1609 +"Dikow2009",88,6,2,"ratchet",0.072,1608 +"Dikow2009",88,6,2,"drift_exact",0.047,1608 +"Dikow2009",88,6,2,"drift_union",0.038,1609 +"Dikow2009",88,6,4,"ratchet",0.116,1608 +"Dikow2009",88,6,4,"drift_exact",0.099,1607 +"Dikow2009",88,6,4,"drift_union",0.093,1607 +"Dikow2009",88,6,8,"ratchet",0.209,1608 +"Dikow2009",88,6,8,"drift_exact",0.179,1607 +"Dikow2009",88,6,8,"drift_union",0.189,1607 +"Dikow2009",88,6,16,"ratchet",0.401,1608 +"Dikow2009",88,6,16,"drift_exact",0.333,1607 +"Dikow2009",88,6,16,"drift_union",0.399,1607 +"Dikow2009",88,7,1,"ratchet",0.047,1608 +"Dikow2009",88,7,1,"drift_exact",0.046,1608 +"Dikow2009",88,7,1,"drift_union",0.034,1608 +"Dikow2009",88,7,2,"ratchet",0.076,1608 +"Dikow2009",88,7,2,"drift_exact",0.06,1607 +"Dikow2009",88,7,2,"drift_union",0.047,1607 +"Dikow2009",88,7,4,"ratchet",0.123,1607 +"Dikow2009",88,7,4,"drift_exact",0.113,1607 +"Dikow2009",88,7,4,"drift_union",0.108,1607 +"Dikow2009",88,7,8,"ratchet",0.239,1606 +"Dikow2009",88,7,8,"drift_exact",0.213,1607 +"Dikow2009",88,7,8,"drift_union",0.216,1607 +"Dikow2009",88,7,16,"ratchet",0.435,1606 +"Dikow2009",88,7,16,"drift_exact",0.443,1606 +"Dikow2009",88,7,16,"drift_union",0.419,1607 +"Dikow2009",88,8,1,"ratchet",0.054,1607 +"Dikow2009",88,8,1,"drift_exact",0.03,1607 +"Dikow2009",88,8,1,"drift_union",0.029,1607 +"Dikow2009",88,8,2,"ratchet",0.077,1607 +"Dikow2009",88,8,2,"drift_exact",0.049,1607 +"Dikow2009",88,8,2,"drift_union",0.048,1607 +"Dikow2009",88,8,4,"ratchet",0.135,1607 +"Dikow2009",88,8,4,"drift_exact",0.098,1607 +"Dikow2009",88,8,4,"drift_union",0.107,1607 +"Dikow2009",88,8,8,"ratchet",0.269,1607 +"Dikow2009",88,8,8,"drift_exact",0.186,1607 +"Dikow2009",88,8,8,"drift_union",0.199,1607 +"Dikow2009",88,8,16,"ratchet",0.484,1607 +"Dikow2009",88,8,16,"drift_exact",0.355,1607 +"Dikow2009",88,8,16,"drift_union",0.4,1607 +"Dikow2009",88,9,1,"ratchet",0.035,1613 +"Dikow2009",88,9,1,"drift_exact",0.047,1610 +"Dikow2009",88,9,1,"drift_union",0.047,1612 +"Dikow2009",88,9,2,"ratchet",0.092,1610 +"Dikow2009",88,9,2,"drift_exact",0.062,1610 +"Dikow2009",88,9,2,"drift_union",0.059,1612 +"Dikow2009",88,9,4,"ratchet",0.144,1610 +"Dikow2009",88,9,4,"drift_exact",0.132,1607 +"Dikow2009",88,9,4,"drift_union",0.109,1612 +"Dikow2009",88,9,8,"ratchet",0.277,1607 +"Dikow2009",88,9,8,"drift_exact",0.205,1607 +"Dikow2009",88,9,8,"drift_union",0.207,1612 +"Dikow2009",88,9,16,"ratchet",0.497,1607 +"Dikow2009",88,9,16,"drift_exact",0.392,1607 +"Dikow2009",88,9,16,"drift_union",0.446,1607 +"Dikow2009",88,10,1,"ratchet",0.058,1611 +"Dikow2009",88,10,1,"drift_exact",0.034,1613 +"Dikow2009",88,10,1,"drift_union",0.049,1611 +"Dikow2009",88,10,2,"ratchet",0.098,1611 +"Dikow2009",88,10,2,"drift_exact",0.048,1612 +"Dikow2009",88,10,2,"drift_union",0.084,1611 +"Dikow2009",88,10,4,"ratchet",0.172,1611 +"Dikow2009",88,10,4,"drift_exact",0.109,1609 +"Dikow2009",88,10,4,"drift_union",0.141,1610 +"Dikow2009",88,10,8,"ratchet",0.305,1608 +"Dikow2009",88,10,8,"drift_exact",0.197,1607 +"Dikow2009",88,10,8,"drift_union",0.224,1609 +"Dikow2009",88,10,16,"ratchet",0.511,1607 +"Dikow2009",88,10,16,"drift_exact",0.373,1607 +"Dikow2009",88,10,16,"drift_union",0.439,1607 +"Dikow2009",88,11,1,"ratchet",0.032,1610 +"Dikow2009",88,11,1,"drift_exact",0.034,1608 +"Dikow2009",88,11,1,"drift_union",0.037,1610 +"Dikow2009",88,11,2,"ratchet",0.075,1609 +"Dikow2009",88,11,2,"drift_exact",0.053,1608 +"Dikow2009",88,11,2,"drift_union",0.051,1610 +"Dikow2009",88,11,4,"ratchet",0.131,1609 +"Dikow2009",88,11,4,"drift_exact",0.107,1606 +"Dikow2009",88,11,4,"drift_union",0.093,1610 +"Dikow2009",88,11,8,"ratchet",0.228,1609 +"Dikow2009",88,11,8,"drift_exact",0.212,1606 +"Dikow2009",88,11,8,"drift_union",0.187,1610 +"Dikow2009",88,11,16,"ratchet",0.424,1609 +"Dikow2009",88,11,16,"drift_exact",0.416,1606 +"Dikow2009",88,11,16,"drift_union",0.371,1610 +"Dikow2009",88,12,1,"ratchet",0.046,1612 +"Dikow2009",88,12,1,"drift_exact",0.047,1609 +"Dikow2009",88,12,1,"drift_union",0.028,1610 +"Dikow2009",88,12,2,"ratchet",0.07,1612 +"Dikow2009",88,12,2,"drift_exact",0.063,1609 +"Dikow2009",88,12,2,"drift_union",0.043,1610 +"Dikow2009",88,12,4,"ratchet",0.114,1612 +"Dikow2009",88,12,4,"drift_exact",0.128,1609 +"Dikow2009",88,12,4,"drift_union",0.098,1607 +"Dikow2009",88,12,8,"ratchet",0.214,1608 +"Dikow2009",88,12,8,"drift_exact",0.24,1608 +"Dikow2009",88,12,8,"drift_union",0.22,1606 +"Dikow2009",88,12,16,"ratchet",0.427,1608 +"Dikow2009",88,12,16,"drift_exact",0.436,1607 +"Dikow2009",88,12,16,"drift_union",0.449,1606 +"Dikow2009",88,13,1,"ratchet",0.042,1609 +"Dikow2009",88,13,1,"drift_exact",0.031,1609 +"Dikow2009",88,13,1,"drift_union",0.031,1609 +"Dikow2009",88,13,2,"ratchet",0.084,1609 +"Dikow2009",88,13,2,"drift_exact",0.054,1609 +"Dikow2009",88,13,2,"drift_union",0.055,1609 +"Dikow2009",88,13,4,"ratchet",0.137,1609 +"Dikow2009",88,13,4,"drift_exact",0.109,1609 +"Dikow2009",88,13,4,"drift_union",0.117,1609 +"Dikow2009",88,13,8,"ratchet",0.245,1609 +"Dikow2009",88,13,8,"drift_exact",0.198,1609 +"Dikow2009",88,13,8,"drift_union",0.217,1609 +"Dikow2009",88,13,16,"ratchet",0.457,1609 +"Dikow2009",88,13,16,"drift_exact",0.391,1609 +"Dikow2009",88,13,16,"drift_union",0.441,1609 +"Dikow2009",88,14,1,"ratchet",0.052,1609 +"Dikow2009",88,14,1,"drift_exact",0.023,1610 +"Dikow2009",88,14,1,"drift_union",0.035,1609 +"Dikow2009",88,14,2,"ratchet",0.088,1609 +"Dikow2009",88,14,2,"drift_exact",0.048,1609 +"Dikow2009",88,14,2,"drift_union",0.05,1609 +"Dikow2009",88,14,4,"ratchet",0.144,1609 +"Dikow2009",88,14,4,"drift_exact",0.094,1609 +"Dikow2009",88,14,4,"drift_union",0.096,1608 +"Dikow2009",88,14,8,"ratchet",0.251,1609 +"Dikow2009",88,14,8,"drift_exact",0.182,1609 +"Dikow2009",88,14,8,"drift_union",0.206,1608 +"Dikow2009",88,14,16,"ratchet",0.456,1609 +"Dikow2009",88,14,16,"drift_exact",0.384,1609 +"Dikow2009",88,14,16,"drift_union",0.431,1608 +"Dikow2009",88,15,1,"ratchet",0.05,1610 +"Dikow2009",88,15,1,"drift_exact",0.028,1610 +"Dikow2009",88,15,1,"drift_union",0.037,1610 +"Dikow2009",88,15,2,"ratchet",0.072,1610 +"Dikow2009",88,15,2,"drift_exact",0.043,1610 +"Dikow2009",88,15,2,"drift_union",0.053,1610 +"Dikow2009",88,15,4,"ratchet",0.133,1608 +"Dikow2009",88,15,4,"drift_exact",0.072,1610 +"Dikow2009",88,15,4,"drift_union",0.089,1610 +"Dikow2009",88,15,8,"ratchet",0.257,1608 +"Dikow2009",88,15,8,"drift_exact",0.139,1610 +"Dikow2009",88,15,8,"drift_union",0.189,1610 +"Dikow2009",88,15,16,"ratchet",0.459,1608 +"Dikow2009",88,15,16,"drift_exact",0.32,1608 +"Dikow2009",88,15,16,"drift_union",0.411,1610 +"Dikow2009",88,16,1,"ratchet",0.035,1609 +"Dikow2009",88,16,1,"drift_exact",0.036,1609 +"Dikow2009",88,16,1,"drift_union",0.028,1609 +"Dikow2009",88,16,2,"ratchet",0.056,1609 +"Dikow2009",88,16,2,"drift_exact",0.049,1609 +"Dikow2009",88,16,2,"drift_union",0.041,1609 +"Dikow2009",88,16,4,"ratchet",0.099,1609 +"Dikow2009",88,16,4,"drift_exact",0.082,1609 +"Dikow2009",88,16,4,"drift_union",0.08,1609 +"Dikow2009",88,16,8,"ratchet",0.196,1609 +"Dikow2009",88,16,8,"drift_exact",0.173,1609 +"Dikow2009",88,16,8,"drift_union",0.185,1607 +"Dikow2009",88,16,16,"ratchet",0.429,1609 +"Dikow2009",88,16,16,"drift_exact",0.429,1609 +"Dikow2009",88,16,16,"drift_union",0.382,1607 +"Dikow2009",88,17,1,"ratchet",0.033,1611 +"Dikow2009",88,17,1,"drift_exact",0.019,1611 +"Dikow2009",88,17,1,"drift_union",0.027,1611 +"Dikow2009",88,17,2,"ratchet",0.054,1611 +"Dikow2009",88,17,2,"drift_exact",0.035,1611 +"Dikow2009",88,17,2,"drift_union",0.043,1611 +"Dikow2009",88,17,4,"ratchet",0.1,1611 +"Dikow2009",88,17,4,"drift_exact",0.077,1611 +"Dikow2009",88,17,4,"drift_union",0.084,1611 +"Dikow2009",88,17,8,"ratchet",0.184,1611 +"Dikow2009",88,17,8,"drift_exact",0.156,1611 +"Dikow2009",88,17,8,"drift_union",0.178,1611 +"Dikow2009",88,17,16,"ratchet",0.407,1611 +"Dikow2009",88,17,16,"drift_exact",0.335,1610 +"Dikow2009",88,17,16,"drift_union",0.349,1611 +"Dikow2009",88,18,1,"ratchet",0.039,1608 +"Dikow2009",88,18,1,"drift_exact",0.038,1609 +"Dikow2009",88,18,1,"drift_union",0.035,1609 +"Dikow2009",88,18,2,"ratchet",0.061,1608 +"Dikow2009",88,18,2,"drift_exact",0.056,1608 +"Dikow2009",88,18,2,"drift_union",0.053,1608 +"Dikow2009",88,18,4,"ratchet",0.109,1608 +"Dikow2009",88,18,4,"drift_exact",0.111,1608 +"Dikow2009",88,18,4,"drift_union",0.109,1608 +"Dikow2009",88,18,8,"ratchet",0.206,1608 +"Dikow2009",88,18,8,"drift_exact",0.209,1608 +"Dikow2009",88,18,8,"drift_union",0.243,1608 +"Dikow2009",88,18,16,"ratchet",0.429,1608 +"Dikow2009",88,18,16,"drift_exact",0.405,1608 +"Dikow2009",88,18,16,"drift_union",0.442,1608 +"Dikow2009",88,19,1,"ratchet",0.04,1607 +"Dikow2009",88,19,1,"drift_exact",0.036,1607 +"Dikow2009",88,19,1,"drift_union",0.034,1607 +"Dikow2009",88,19,2,"ratchet",0.062,1607 +"Dikow2009",88,19,2,"drift_exact",0.047,1607 +"Dikow2009",88,19,2,"drift_union",0.046,1607 +"Dikow2009",88,19,4,"ratchet",0.134,1607 +"Dikow2009",88,19,4,"drift_exact",0.089,1607 +"Dikow2009",88,19,4,"drift_union",0.103,1607 +"Dikow2009",88,19,8,"ratchet",0.224,1607 +"Dikow2009",88,19,8,"drift_exact",0.188,1607 +"Dikow2009",88,19,8,"drift_union",0.208,1607 +"Dikow2009",88,19,16,"ratchet",0.42,1607 +"Dikow2009",88,19,16,"drift_exact",0.386,1607 +"Dikow2009",88,19,16,"drift_union",0.365,1607 +"Dikow2009",88,20,1,"ratchet",0.039,1615 +"Dikow2009",88,20,1,"drift_exact",0.038,1614 +"Dikow2009",88,20,1,"drift_union",0.041,1614 +"Dikow2009",88,20,2,"ratchet",0.077,1611 +"Dikow2009",88,20,2,"drift_exact",0.049,1614 +"Dikow2009",88,20,2,"drift_union",0.059,1614 +"Dikow2009",88,20,4,"ratchet",0.129,1611 +"Dikow2009",88,20,4,"drift_exact",0.117,1610 +"Dikow2009",88,20,4,"drift_union",0.126,1609 +"Dikow2009",88,20,8,"ratchet",0.264,1610 +"Dikow2009",88,20,8,"drift_exact",0.218,1610 +"Dikow2009",88,20,8,"drift_union",0.207,1609 +"Dikow2009",88,20,16,"ratchet",0.449,1609 +"Dikow2009",88,20,16,"drift_exact",0.376,1610 +"Dikow2009",88,20,16,"drift_union",0.36,1609 diff --git a/dev/profiling/route3-data/route3_Zanol2014.csv b/dev/profiling/route3-data/route3_Zanol2014.csv new file mode 100644 index 000000000..21f01c142 --- /dev/null +++ b/dev/profiling/route3-data/route3_Zanol2014.csv @@ -0,0 +1,301 @@ +"dataset","nTip","seed","cycles","arm","wall_s","score" +"Zanol2014",74,1,1,"ratchet",0.033,1268 +"Zanol2014",74,1,1,"drift_exact",0.026,1266 +"Zanol2014",74,1,1,"drift_union",0.044,1272 +"Zanol2014",74,1,2,"ratchet",0.054,1267 +"Zanol2014",74,1,2,"drift_exact",0.034,1266 +"Zanol2014",74,1,2,"drift_union",0.051,1272 +"Zanol2014",74,1,4,"ratchet",0.093,1267 +"Zanol2014",74,1,4,"drift_exact",0.053,1266 +"Zanol2014",74,1,4,"drift_union",0.142,1266 +"Zanol2014",74,1,8,"ratchet",0.152,1266 +"Zanol2014",74,1,8,"drift_exact",0.102,1266 +"Zanol2014",74,1,8,"drift_union",0.274,1263 +"Zanol2014",74,1,16,"ratchet",0.281,1266 +"Zanol2014",74,1,16,"drift_exact",0.216,1266 +"Zanol2014",74,1,16,"drift_union",0.513,1262 +"Zanol2014",74,2,1,"ratchet",0.02,1269 +"Zanol2014",74,2,1,"drift_exact",0.028,1267 +"Zanol2014",74,2,1,"drift_union",0.064,1269 +"Zanol2014",74,2,2,"ratchet",0.046,1266 +"Zanol2014",74,2,2,"drift_exact",0.035,1267 +"Zanol2014",74,2,2,"drift_union",0.071,1269 +"Zanol2014",74,2,4,"ratchet",0.091,1263 +"Zanol2014",74,2,4,"drift_exact",0.053,1267 +"Zanol2014",74,2,4,"drift_union",0.134,1263 +"Zanol2014",74,2,8,"ratchet",0.147,1263 +"Zanol2014",74,2,8,"drift_exact",0.111,1267 +"Zanol2014",74,2,8,"drift_union",0.241,1263 +"Zanol2014",74,2,16,"ratchet",0.251,1263 +"Zanol2014",74,2,16,"drift_exact",0.215,1264 +"Zanol2014",74,2,16,"drift_union",0.492,1263 +"Zanol2014",74,3,1,"ratchet",0.02,1263 +"Zanol2014",74,3,1,"drift_exact",0.013,1263 +"Zanol2014",74,3,1,"drift_union",0.057,1263 +"Zanol2014",74,3,2,"ratchet",0.035,1263 +"Zanol2014",74,3,2,"drift_exact",0.024,1262 +"Zanol2014",74,3,2,"drift_union",0.069,1262 +"Zanol2014",74,3,4,"ratchet",0.063,1262 +"Zanol2014",74,3,4,"drift_exact",0.041,1262 +"Zanol2014",74,3,4,"drift_union",0.122,1262 +"Zanol2014",74,3,8,"ratchet",0.117,1262 +"Zanol2014",74,3,8,"drift_exact",0.091,1262 +"Zanol2014",74,3,8,"drift_union",0.244,1262 +"Zanol2014",74,3,16,"ratchet",0.223,1262 +"Zanol2014",74,3,16,"drift_exact",0.191,1262 +"Zanol2014",74,3,16,"drift_union",0.541,1262 +"Zanol2014",74,4,1,"ratchet",0.022,1265 +"Zanol2014",74,4,1,"drift_exact",0.013,1265 +"Zanol2014",74,4,1,"drift_union",0.059,1265 +"Zanol2014",74,4,2,"ratchet",0.039,1265 +"Zanol2014",74,4,2,"drift_exact",0.021,1265 +"Zanol2014",74,4,2,"drift_union",0.066,1265 +"Zanol2014",74,4,4,"ratchet",0.069,1265 +"Zanol2014",74,4,4,"drift_exact",0.039,1265 +"Zanol2014",74,4,4,"drift_union",0.133,1265 +"Zanol2014",74,4,8,"ratchet",0.136,1265 +"Zanol2014",74,4,8,"drift_exact",0.086,1262 +"Zanol2014",74,4,8,"drift_union",0.244,1265 +"Zanol2014",74,4,16,"ratchet",0.247,1265 +"Zanol2014",74,4,16,"drift_exact",0.188,1262 +"Zanol2014",74,4,16,"drift_union",0.491,1264 +"Zanol2014",74,5,1,"ratchet",0.02,1262 +"Zanol2014",74,5,1,"drift_exact",0.015,1262 +"Zanol2014",74,5,1,"drift_union",0.088,1262 +"Zanol2014",74,5,2,"ratchet",0.033,1262 +"Zanol2014",74,5,2,"drift_exact",0.023,1262 +"Zanol2014",74,5,2,"drift_union",0.096,1262 +"Zanol2014",74,5,4,"ratchet",0.057,1262 +"Zanol2014",74,5,4,"drift_exact",0.052,1262 +"Zanol2014",74,5,4,"drift_union",0.184,1262 +"Zanol2014",74,5,8,"ratchet",0.121,1262 +"Zanol2014",74,5,8,"drift_exact",0.111,1262 +"Zanol2014",74,5,8,"drift_union",0.303,1262 +"Zanol2014",74,5,16,"ratchet",0.23,1262 +"Zanol2014",74,5,16,"drift_exact",0.207,1262 +"Zanol2014",74,5,16,"drift_union",0.544,1262 +"Zanol2014",74,6,1,"ratchet",0.03,1268 +"Zanol2014",74,6,1,"drift_exact",0.018,1265 +"Zanol2014",74,6,1,"drift_union",0.073,1265 +"Zanol2014",74,6,2,"ratchet",0.055,1264 +"Zanol2014",74,6,2,"drift_exact",0.024,1265 +"Zanol2014",74,6,2,"drift_union",0.079,1265 +"Zanol2014",74,6,4,"ratchet",0.093,1264 +"Zanol2014",74,6,4,"drift_exact",0.048,1262 +"Zanol2014",74,6,4,"drift_union",0.16,1265 +"Zanol2014",74,6,8,"ratchet",0.16,1262 +"Zanol2014",74,6,8,"drift_exact",0.096,1262 +"Zanol2014",74,6,8,"drift_union",0.332,1265 +"Zanol2014",74,6,16,"ratchet",0.281,1262 +"Zanol2014",74,6,16,"drift_exact",0.191,1262 +"Zanol2014",74,6,16,"drift_union",0.583,1263 +"Zanol2014",74,7,1,"ratchet",0.033,1266 +"Zanol2014",74,7,1,"drift_exact",0.017,1266 +"Zanol2014",74,7,1,"drift_union",0.032,1268 +"Zanol2014",74,7,2,"ratchet",0.045,1266 +"Zanol2014",74,7,2,"drift_exact",0.024,1266 +"Zanol2014",74,7,2,"drift_union",0.039,1268 +"Zanol2014",74,7,4,"ratchet",0.07,1266 +"Zanol2014",74,7,4,"drift_exact",0.045,1264 +"Zanol2014",74,7,4,"drift_union",0.102,1262 +"Zanol2014",74,7,8,"ratchet",0.137,1266 +"Zanol2014",74,7,8,"drift_exact",0.089,1264 +"Zanol2014",74,7,8,"drift_union",0.27,1262 +"Zanol2014",74,7,16,"ratchet",0.264,1265 +"Zanol2014",74,7,16,"drift_exact",0.193,1264 +"Zanol2014",74,7,16,"drift_union",0.525,1262 +"Zanol2014",74,8,1,"ratchet",0.02,1267 +"Zanol2014",74,8,1,"drift_exact",0.019,1267 +"Zanol2014",74,8,1,"drift_union",0.067,1264 +"Zanol2014",74,8,2,"ratchet",0.034,1267 +"Zanol2014",74,8,2,"drift_exact",0.025,1267 +"Zanol2014",74,8,2,"drift_union",0.074,1264 +"Zanol2014",74,8,4,"ratchet",0.064,1267 +"Zanol2014",74,8,4,"drift_exact",0.043,1266 +"Zanol2014",74,8,4,"drift_union",0.145,1264 +"Zanol2014",74,8,8,"ratchet",0.142,1264 +"Zanol2014",74,8,8,"drift_exact",0.09,1265 +"Zanol2014",74,8,8,"drift_union",0.291,1264 +"Zanol2014",74,8,16,"ratchet",0.281,1264 +"Zanol2014",74,8,16,"drift_exact",0.175,1264 +"Zanol2014",74,8,16,"drift_union",0.559,1264 +"Zanol2014",74,9,1,"ratchet",0.021,1273 +"Zanol2014",74,9,1,"drift_exact",0.021,1272 +"Zanol2014",74,9,1,"drift_union",0.062,1272 +"Zanol2014",74,9,2,"ratchet",0.048,1273 +"Zanol2014",74,9,2,"drift_exact",0.028,1272 +"Zanol2014",74,9,2,"drift_union",0.068,1272 +"Zanol2014",74,9,4,"ratchet",0.08,1269 +"Zanol2014",74,9,4,"drift_exact",0.058,1269 +"Zanol2014",74,9,4,"drift_union",0.14,1269 +"Zanol2014",74,9,8,"ratchet",0.137,1269 +"Zanol2014",74,9,8,"drift_exact",0.098,1269 +"Zanol2014",74,9,8,"drift_union",0.292,1265 +"Zanol2014",74,9,16,"ratchet",0.275,1268 +"Zanol2014",74,9,16,"drift_exact",0.209,1262 +"Zanol2014",74,9,16,"drift_union",0.551,1265 +"Zanol2014",74,10,1,"ratchet",0.027,1269 +"Zanol2014",74,10,1,"drift_exact",0.024,1264 +"Zanol2014",74,10,1,"drift_union",0.061,1271 +"Zanol2014",74,10,2,"ratchet",0.056,1263 +"Zanol2014",74,10,2,"drift_exact",0.033,1263 +"Zanol2014",74,10,2,"drift_union",0.071,1271 +"Zanol2014",74,10,4,"ratchet",0.093,1263 +"Zanol2014",74,10,4,"drift_exact",0.055,1263 +"Zanol2014",74,10,4,"drift_union",0.167,1265 +"Zanol2014",74,10,8,"ratchet",0.162,1263 +"Zanol2014",74,10,8,"drift_exact",0.104,1263 +"Zanol2014",74,10,8,"drift_union",0.295,1264 +"Zanol2014",74,10,16,"ratchet",0.277,1263 +"Zanol2014",74,10,16,"drift_exact",0.225,1262 +"Zanol2014",74,10,16,"drift_union",0.558,1262 +"Zanol2014",74,11,1,"ratchet",0.04,1265 +"Zanol2014",74,11,1,"drift_exact",0.019,1265 +"Zanol2014",74,11,1,"drift_union",0.049,1269 +"Zanol2014",74,11,2,"ratchet",0.061,1265 +"Zanol2014",74,11,2,"drift_exact",0.026,1265 +"Zanol2014",74,11,2,"drift_union",0.057,1269 +"Zanol2014",74,11,4,"ratchet",0.096,1265 +"Zanol2014",74,11,4,"drift_exact",0.046,1265 +"Zanol2014",74,11,4,"drift_union",0.116,1266 +"Zanol2014",74,11,8,"ratchet",0.152,1265 +"Zanol2014",74,11,8,"drift_exact",0.092,1264 +"Zanol2014",74,11,8,"drift_union",0.242,1263 +"Zanol2014",74,11,16,"ratchet",0.277,1265 +"Zanol2014",74,11,16,"drift_exact",0.199,1262 +"Zanol2014",74,11,16,"drift_union",0.532,1263 +"Zanol2014",74,12,1,"ratchet",0.029,1264 +"Zanol2014",74,12,1,"drift_exact",0.02,1262 +"Zanol2014",74,12,1,"drift_union",0.062,1264 +"Zanol2014",74,12,2,"ratchet",0.045,1262 +"Zanol2014",74,12,2,"drift_exact",0.031,1262 +"Zanol2014",74,12,2,"drift_union",0.072,1264 +"Zanol2014",74,12,4,"ratchet",0.069,1262 +"Zanol2014",74,12,4,"drift_exact",0.057,1262 +"Zanol2014",74,12,4,"drift_union",0.125,1264 +"Zanol2014",74,12,8,"ratchet",0.122,1262 +"Zanol2014",74,12,8,"drift_exact",0.118,1262 +"Zanol2014",74,12,8,"drift_union",0.242,1264 +"Zanol2014",74,12,16,"ratchet",0.238,1262 +"Zanol2014",74,12,16,"drift_exact",0.223,1262 +"Zanol2014",74,12,16,"drift_union",0.486,1262 +"Zanol2014",74,13,1,"ratchet",0.039,1267 +"Zanol2014",74,13,1,"drift_exact",0.033,1265 +"Zanol2014",74,13,1,"drift_union",0.085,1264 +"Zanol2014",74,13,2,"ratchet",0.066,1265 +"Zanol2014",74,13,2,"drift_exact",0.041,1265 +"Zanol2014",74,13,2,"drift_union",0.092,1264 +"Zanol2014",74,13,4,"ratchet",0.099,1265 +"Zanol2014",74,13,4,"drift_exact",0.062,1265 +"Zanol2014",74,13,4,"drift_union",0.169,1264 +"Zanol2014",74,13,8,"ratchet",0.152,1265 +"Zanol2014",74,13,8,"drift_exact",0.098,1263 +"Zanol2014",74,13,8,"drift_union",0.326,1263 +"Zanol2014",74,13,16,"ratchet",0.296,1264 +"Zanol2014",74,13,16,"drift_exact",0.19,1263 +"Zanol2014",74,13,16,"drift_union",0.615,1262 +"Zanol2014",74,14,1,"ratchet",0.02,1265 +"Zanol2014",74,14,1,"drift_exact",0.016,1265 +"Zanol2014",74,14,1,"drift_union",0.066,1264 +"Zanol2014",74,14,2,"ratchet",0.041,1265 +"Zanol2014",74,14,2,"drift_exact",0.024,1265 +"Zanol2014",74,14,2,"drift_union",0.075,1263 +"Zanol2014",74,14,4,"ratchet",0.073,1265 +"Zanol2014",74,14,4,"drift_exact",0.053,1262 +"Zanol2014",74,14,4,"drift_union",0.144,1262 +"Zanol2014",74,14,8,"ratchet",0.132,1264 +"Zanol2014",74,14,8,"drift_exact",0.102,1262 +"Zanol2014",74,14,8,"drift_union",0.263,1262 +"Zanol2014",74,14,16,"ratchet",0.247,1264 +"Zanol2014",74,14,16,"drift_exact",0.21,1262 +"Zanol2014",74,14,16,"drift_union",0.536,1262 +"Zanol2014",74,15,1,"ratchet",0.03,1273 +"Zanol2014",74,15,1,"drift_exact",0.027,1269 +"Zanol2014",74,15,1,"drift_union",0.048,1271 +"Zanol2014",74,15,2,"ratchet",0.052,1272 +"Zanol2014",74,15,2,"drift_exact",0.034,1269 +"Zanol2014",74,15,2,"drift_union",0.056,1271 +"Zanol2014",74,15,4,"ratchet",0.097,1271 +"Zanol2014",74,15,4,"drift_exact",0.061,1269 +"Zanol2014",74,15,4,"drift_union",0.115,1269 +"Zanol2014",74,15,8,"ratchet",0.171,1269 +"Zanol2014",74,15,8,"drift_exact",0.109,1267 +"Zanol2014",74,15,8,"drift_union",0.251,1269 +"Zanol2014",74,15,16,"ratchet",0.29,1269 +"Zanol2014",74,15,16,"drift_exact",0.215,1264 +"Zanol2014",74,15,16,"drift_union",0.509,1262 +"Zanol2014",74,16,1,"ratchet",0.02,1270 +"Zanol2014",74,16,1,"drift_exact",0.017,1270 +"Zanol2014",74,16,1,"drift_union",0.066,1266 +"Zanol2014",74,16,2,"ratchet",0.032,1270 +"Zanol2014",74,16,2,"drift_exact",0.024,1270 +"Zanol2014",74,16,2,"drift_union",0.074,1266 +"Zanol2014",74,16,4,"ratchet",0.061,1270 +"Zanol2014",74,16,4,"drift_exact",0.04,1270 +"Zanol2014",74,16,4,"drift_union",0.167,1262 +"Zanol2014",74,16,8,"ratchet",0.109,1270 +"Zanol2014",74,16,8,"drift_exact",0.099,1267 +"Zanol2014",74,16,8,"drift_union",0.32,1262 +"Zanol2014",74,16,16,"ratchet",0.212,1267 +"Zanol2014",74,16,16,"drift_exact",0.184,1266 +"Zanol2014",74,16,16,"drift_union",0.577,1262 +"Zanol2014",74,17,1,"ratchet",0.02,1266 +"Zanol2014",74,17,1,"drift_exact",0.016,1266 +"Zanol2014",74,17,1,"drift_union",0.079,1263 +"Zanol2014",74,17,2,"ratchet",0.032,1266 +"Zanol2014",74,17,2,"drift_exact",0.023,1266 +"Zanol2014",74,17,2,"drift_union",0.087,1263 +"Zanol2014",74,17,4,"ratchet",0.056,1266 +"Zanol2014",74,17,4,"drift_exact",0.055,1265 +"Zanol2014",74,17,4,"drift_union",0.129,1263 +"Zanol2014",74,17,8,"ratchet",0.145,1262 +"Zanol2014",74,17,8,"drift_exact",0.106,1264 +"Zanol2014",74,17,8,"drift_union",0.279,1262 +"Zanol2014",74,17,16,"ratchet",0.264,1262 +"Zanol2014",74,17,16,"drift_exact",0.191,1262 +"Zanol2014",74,17,16,"drift_union",0.558,1262 +"Zanol2014",74,18,1,"ratchet",0.02,1264 +"Zanol2014",74,18,1,"drift_exact",0.018,1263 +"Zanol2014",74,18,1,"drift_union",0.055,1264 +"Zanol2014",74,18,2,"ratchet",0.041,1264 +"Zanol2014",74,18,2,"drift_exact",0.026,1263 +"Zanol2014",74,18,2,"drift_union",0.062,1264 +"Zanol2014",74,18,4,"ratchet",0.066,1264 +"Zanol2014",74,18,4,"drift_exact",0.058,1262 +"Zanol2014",74,18,4,"drift_union",0.118,1264 +"Zanol2014",74,18,8,"ratchet",0.139,1264 +"Zanol2014",74,18,8,"drift_exact",0.101,1262 +"Zanol2014",74,18,8,"drift_union",0.233,1264 +"Zanol2014",74,18,16,"ratchet",0.264,1264 +"Zanol2014",74,18,16,"drift_exact",0.195,1262 +"Zanol2014",74,18,16,"drift_union",0.491,1263 +"Zanol2014",74,19,1,"ratchet",0.033,1266 +"Zanol2014",74,19,1,"drift_exact",0.023,1264 +"Zanol2014",74,19,1,"drift_union",0.068,1267 +"Zanol2014",74,19,2,"ratchet",0.06,1266 +"Zanol2014",74,19,2,"drift_exact",0.032,1264 +"Zanol2014",74,19,2,"drift_union",0.082,1267 +"Zanol2014",74,19,4,"ratchet",0.085,1266 +"Zanol2014",74,19,4,"drift_exact",0.051,1264 +"Zanol2014",74,19,4,"drift_union",0.176,1266 +"Zanol2014",74,19,8,"ratchet",0.14,1264 +"Zanol2014",74,19,8,"drift_exact",0.096,1263 +"Zanol2014",74,19,8,"drift_union",0.315,1265 +"Zanol2014",74,19,16,"ratchet",0.259,1264 +"Zanol2014",74,19,16,"drift_exact",0.191,1263 +"Zanol2014",74,19,16,"drift_union",0.529,1263 +"Zanol2014",74,20,1,"ratchet",0.032,1265 +"Zanol2014",74,20,1,"drift_exact",0.033,1263 +"Zanol2014",74,20,1,"drift_union",0.074,1266 +"Zanol2014",74,20,2,"ratchet",0.046,1265 +"Zanol2014",74,20,2,"drift_exact",0.042,1263 +"Zanol2014",74,20,2,"drift_union",0.081,1266 +"Zanol2014",74,20,4,"ratchet",0.078,1263 +"Zanol2014",74,20,4,"drift_exact",0.068,1263 +"Zanol2014",74,20,4,"drift_union",0.161,1265 +"Zanol2014",74,20,8,"ratchet",0.129,1263 +"Zanol2014",74,20,8,"drift_exact",0.107,1263 +"Zanol2014",74,20,8,"drift_union",0.278,1265 +"Zanol2014",74,20,16,"ratchet",0.243,1263 +"Zanol2014",74,20,16,"drift_exact",0.205,1263 +"Zanol2014",74,20,16,"drift_union",0.527,1265 diff --git a/dev/profiling/route3-data/route3_Zhu2013.csv b/dev/profiling/route3-data/route3_Zhu2013.csv new file mode 100644 index 000000000..92e834161 --- /dev/null +++ b/dev/profiling/route3-data/route3_Zhu2013.csv @@ -0,0 +1,301 @@ +"dataset","nTip","seed","cycles","arm","wall_s","score" +"Zhu2013",75,1,1,"ratchet",0.013,631 +"Zhu2013",75,1,1,"drift_exact",0.014,629 +"Zhu2013",75,1,1,"drift_union",0.039,630 +"Zhu2013",75,1,2,"ratchet",0.023,631 +"Zhu2013",75,1,2,"drift_exact",0.019,629 +"Zhu2013",75,1,2,"drift_union",0.044,630 +"Zhu2013",75,1,4,"ratchet",0.041,631 +"Zhu2013",75,1,4,"drift_exact",0.036,627 +"Zhu2013",75,1,4,"drift_union",0.09,629 +"Zhu2013",75,1,8,"ratchet",0.088,627 +"Zhu2013",75,1,8,"drift_exact",0.07,627 +"Zhu2013",75,1,8,"drift_union",0.202,627 +"Zhu2013",75,1,16,"ratchet",0.161,627 +"Zhu2013",75,1,16,"drift_exact",0.136,625 +"Zhu2013",75,1,16,"drift_union",0.39,625 +"Zhu2013",75,2,1,"ratchet",0.019,629 +"Zhu2013",75,2,1,"drift_exact",0.009,627 +"Zhu2013",75,2,1,"drift_union",0.046,629 +"Zhu2013",75,2,2,"ratchet",0.04,625 +"Zhu2013",75,2,2,"drift_exact",0.015,627 +"Zhu2013",75,2,2,"drift_union",0.055,627 +"Zhu2013",75,2,4,"ratchet",0.062,625 +"Zhu2013",75,2,4,"drift_exact",0.028,627 +"Zhu2013",75,2,4,"drift_union",0.105,626 +"Zhu2013",75,2,8,"ratchet",0.095,625 +"Zhu2013",75,2,8,"drift_exact",0.057,625 +"Zhu2013",75,2,8,"drift_union",0.194,625 +"Zhu2013",75,2,16,"ratchet",0.169,625 +"Zhu2013",75,2,16,"drift_exact",0.114,625 +"Zhu2013",75,2,16,"drift_union",0.397,625 +"Zhu2013",75,3,1,"ratchet",0.014,626 +"Zhu2013",75,3,1,"drift_exact",0.016,626 +"Zhu2013",75,3,1,"drift_union",0.029,626 +"Zhu2013",75,3,2,"ratchet",0.022,626 +"Zhu2013",75,3,2,"drift_exact",0.024,626 +"Zhu2013",75,3,2,"drift_union",0.037,626 +"Zhu2013",75,3,4,"ratchet",0.038,626 +"Zhu2013",75,3,4,"drift_exact",0.042,626 +"Zhu2013",75,3,4,"drift_union",0.088,626 +"Zhu2013",75,3,8,"ratchet",0.075,625 +"Zhu2013",75,3,8,"drift_exact",0.079,625 +"Zhu2013",75,3,8,"drift_union",0.19,625 +"Zhu2013",75,3,16,"ratchet",0.144,625 +"Zhu2013",75,3,16,"drift_exact",0.141,625 +"Zhu2013",75,3,16,"drift_union",0.409,625 +"Zhu2013",75,4,1,"ratchet",0.013,627 +"Zhu2013",75,4,1,"drift_exact",0.01,627 +"Zhu2013",75,4,1,"drift_union",0.036,627 +"Zhu2013",75,4,2,"ratchet",0.031,624 +"Zhu2013",75,4,2,"drift_exact",0.014,627 +"Zhu2013",75,4,2,"drift_union",0.04,627 +"Zhu2013",75,4,4,"ratchet",0.047,624 +"Zhu2013",75,4,4,"drift_exact",0.04,625 +"Zhu2013",75,4,4,"drift_union",0.086,627 +"Zhu2013",75,4,8,"ratchet",0.081,624 +"Zhu2013",75,4,8,"drift_exact",0.077,625 +"Zhu2013",75,4,8,"drift_union",0.185,627 +"Zhu2013",75,4,16,"ratchet",0.145,624 +"Zhu2013",75,4,16,"drift_exact",0.129,625 +"Zhu2013",75,4,16,"drift_union",0.362,625 +"Zhu2013",75,5,1,"ratchet",0.013,626 +"Zhu2013",75,5,1,"drift_exact",0.012,626 +"Zhu2013",75,5,1,"drift_union",0.042,626 +"Zhu2013",75,5,2,"ratchet",0.022,626 +"Zhu2013",75,5,2,"drift_exact",0.018,626 +"Zhu2013",75,5,2,"drift_union",0.049,626 +"Zhu2013",75,5,4,"ratchet",0.039,626 +"Zhu2013",75,5,4,"drift_exact",0.036,626 +"Zhu2013",75,5,4,"drift_union",0.112,626 +"Zhu2013",75,5,8,"ratchet",0.072,626 +"Zhu2013",75,5,8,"drift_exact",0.067,626 +"Zhu2013",75,5,8,"drift_union",0.196,626 +"Zhu2013",75,5,16,"ratchet",0.144,624 +"Zhu2013",75,5,16,"drift_exact",0.134,624 +"Zhu2013",75,5,16,"drift_union",0.407,624 +"Zhu2013",75,6,1,"ratchet",0.019,625 +"Zhu2013",75,6,1,"drift_exact",0.009,625 +"Zhu2013",75,6,1,"drift_union",0.036,626 +"Zhu2013",75,6,2,"ratchet",0.027,625 +"Zhu2013",75,6,2,"drift_exact",0.014,625 +"Zhu2013",75,6,2,"drift_union",0.04,626 +"Zhu2013",75,6,4,"ratchet",0.043,625 +"Zhu2013",75,6,4,"drift_exact",0.024,625 +"Zhu2013",75,6,4,"drift_union",0.095,626 +"Zhu2013",75,6,8,"ratchet",0.076,625 +"Zhu2013",75,6,8,"drift_exact",0.056,625 +"Zhu2013",75,6,8,"drift_union",0.184,626 +"Zhu2013",75,6,16,"ratchet",0.158,625 +"Zhu2013",75,6,16,"drift_exact",0.111,624 +"Zhu2013",75,6,16,"drift_union",0.349,626 +"Zhu2013",75,7,1,"ratchet",0.013,626 +"Zhu2013",75,7,1,"drift_exact",0.012,626 +"Zhu2013",75,7,1,"drift_union",0.038,626 +"Zhu2013",75,7,2,"ratchet",0.022,626 +"Zhu2013",75,7,2,"drift_exact",0.016,626 +"Zhu2013",75,7,2,"drift_union",0.043,626 +"Zhu2013",75,7,4,"ratchet",0.047,626 +"Zhu2013",75,7,4,"drift_exact",0.035,626 +"Zhu2013",75,7,4,"drift_union",0.102,626 +"Zhu2013",75,7,8,"ratchet",0.084,626 +"Zhu2013",75,7,8,"drift_exact",0.07,625 +"Zhu2013",75,7,8,"drift_union",0.216,626 +"Zhu2013",75,7,16,"ratchet",0.166,626 +"Zhu2013",75,7,16,"drift_exact",0.131,625 +"Zhu2013",75,7,16,"drift_union",0.428,625 +"Zhu2013",75,8,1,"ratchet",0.014,628 +"Zhu2013",75,8,1,"drift_exact",0.01,628 +"Zhu2013",75,8,1,"drift_union",0.053,628 +"Zhu2013",75,8,2,"ratchet",0.024,628 +"Zhu2013",75,8,2,"drift_exact",0.016,628 +"Zhu2013",75,8,2,"drift_union",0.059,628 +"Zhu2013",75,8,4,"ratchet",0.039,628 +"Zhu2013",75,8,4,"drift_exact",0.031,628 +"Zhu2013",75,8,4,"drift_union",0.11,628 +"Zhu2013",75,8,8,"ratchet",0.07,628 +"Zhu2013",75,8,8,"drift_exact",0.069,627 +"Zhu2013",75,8,8,"drift_union",0.186,628 +"Zhu2013",75,8,16,"ratchet",0.143,628 +"Zhu2013",75,8,16,"drift_exact",0.129,627 +"Zhu2013",75,8,16,"drift_union",0.351,625 +"Zhu2013",75,9,1,"ratchet",0.013,632 +"Zhu2013",75,9,1,"drift_exact",0.007,631 +"Zhu2013",75,9,1,"drift_union",0.037,626 +"Zhu2013",75,9,2,"ratchet",0.022,632 +"Zhu2013",75,9,2,"drift_exact",0.012,631 +"Zhu2013",75,9,2,"drift_union",0.046,626 +"Zhu2013",75,9,4,"ratchet",0.046,631 +"Zhu2013",75,9,4,"drift_exact",0.027,628 +"Zhu2013",75,9,4,"drift_union",0.079,626 +"Zhu2013",75,9,8,"ratchet",0.082,630 +"Zhu2013",75,9,8,"drift_exact",0.056,628 +"Zhu2013",75,9,8,"drift_union",0.178,626 +"Zhu2013",75,9,16,"ratchet",0.162,626 +"Zhu2013",75,9,16,"drift_exact",0.121,625 +"Zhu2013",75,9,16,"drift_union",0.376,626 +"Zhu2013",75,10,1,"ratchet",0.013,627 +"Zhu2013",75,10,1,"drift_exact",0.014,627 +"Zhu2013",75,10,1,"drift_union",0.049,627 +"Zhu2013",75,10,2,"ratchet",0.021,627 +"Zhu2013",75,10,2,"drift_exact",0.019,627 +"Zhu2013",75,10,2,"drift_union",0.054,627 +"Zhu2013",75,10,4,"ratchet",0.041,627 +"Zhu2013",75,10,4,"drift_exact",0.031,626 +"Zhu2013",75,10,4,"drift_union",0.119,626 +"Zhu2013",75,10,8,"ratchet",0.078,627 +"Zhu2013",75,10,8,"drift_exact",0.057,626 +"Zhu2013",75,10,8,"drift_union",0.228,626 +"Zhu2013",75,10,16,"ratchet",0.154,626 +"Zhu2013",75,10,16,"drift_exact",0.116,626 +"Zhu2013",75,10,16,"drift_union",0.423,626 +"Zhu2013",75,11,1,"ratchet",0.013,629 +"Zhu2013",75,11,1,"drift_exact",0.015,625 +"Zhu2013",75,11,1,"drift_union",0.045,629 +"Zhu2013",75,11,2,"ratchet",0.023,629 +"Zhu2013",75,11,2,"drift_exact",0.021,625 +"Zhu2013",75,11,2,"drift_union",0.05,629 +"Zhu2013",75,11,4,"ratchet",0.045,629 +"Zhu2013",75,11,4,"drift_exact",0.043,625 +"Zhu2013",75,11,4,"drift_union",0.079,627 +"Zhu2013",75,11,8,"ratchet",0.094,624 +"Zhu2013",75,11,8,"drift_exact",0.082,625 +"Zhu2013",75,11,8,"drift_union",0.184,626 +"Zhu2013",75,11,16,"ratchet",0.16,624 +"Zhu2013",75,11,16,"drift_exact",0.149,625 +"Zhu2013",75,11,16,"drift_union",0.37,626 +"Zhu2013",75,12,1,"ratchet",0.013,629 +"Zhu2013",75,12,1,"drift_exact",0.016,628 +"Zhu2013",75,12,1,"drift_union",0.037,626 +"Zhu2013",75,12,2,"ratchet",0.02,629 +"Zhu2013",75,12,2,"drift_exact",0.02,628 +"Zhu2013",75,12,2,"drift_union",0.043,626 +"Zhu2013",75,12,4,"ratchet",0.04,628 +"Zhu2013",75,12,4,"drift_exact",0.035,628 +"Zhu2013",75,12,4,"drift_union",0.086,626 +"Zhu2013",75,12,8,"ratchet",0.078,627 +"Zhu2013",75,12,8,"drift_exact",0.074,627 +"Zhu2013",75,12,8,"drift_union",0.18,626 +"Zhu2013",75,12,16,"ratchet",0.151,627 +"Zhu2013",75,12,16,"drift_exact",0.139,626 +"Zhu2013",75,12,16,"drift_union",0.367,625 +"Zhu2013",75,13,1,"ratchet",0.017,625 +"Zhu2013",75,13,1,"drift_exact",0.014,625 +"Zhu2013",75,13,1,"drift_union",0.049,625 +"Zhu2013",75,13,2,"ratchet",0.029,625 +"Zhu2013",75,13,2,"drift_exact",0.018,625 +"Zhu2013",75,13,2,"drift_union",0.053,625 +"Zhu2013",75,13,4,"ratchet",0.045,625 +"Zhu2013",75,13,4,"drift_exact",0.042,625 +"Zhu2013",75,13,4,"drift_union",0.095,625 +"Zhu2013",75,13,8,"ratchet",0.077,625 +"Zhu2013",75,13,8,"drift_exact",0.078,625 +"Zhu2013",75,13,8,"drift_union",0.202,625 +"Zhu2013",75,13,16,"ratchet",0.142,625 +"Zhu2013",75,13,16,"drift_exact",0.138,625 +"Zhu2013",75,13,16,"drift_union",0.378,625 +"Zhu2013",75,14,1,"ratchet",0.016,627 +"Zhu2013",75,14,1,"drift_exact",0.008,628 +"Zhu2013",75,14,1,"drift_union",0.044,628 +"Zhu2013",75,14,2,"ratchet",0.034,627 +"Zhu2013",75,14,2,"drift_exact",0.014,628 +"Zhu2013",75,14,2,"drift_union",0.05,628 +"Zhu2013",75,14,4,"ratchet",0.058,625 +"Zhu2013",75,14,4,"drift_exact",0.031,627 +"Zhu2013",75,14,4,"drift_union",0.091,628 +"Zhu2013",75,14,8,"ratchet",0.092,625 +"Zhu2013",75,14,8,"drift_exact",0.065,625 +"Zhu2013",75,14,8,"drift_union",0.208,624 +"Zhu2013",75,14,16,"ratchet",0.173,625 +"Zhu2013",75,14,16,"drift_exact",0.127,625 +"Zhu2013",75,14,16,"drift_union",0.389,624 +"Zhu2013",75,15,1,"ratchet",0.013,638 +"Zhu2013",75,15,1,"drift_exact",0.012,629 +"Zhu2013",75,15,1,"drift_union",0.041,636 +"Zhu2013",75,15,2,"ratchet",0.027,636 +"Zhu2013",75,15,2,"drift_exact",0.017,629 +"Zhu2013",75,15,2,"drift_union",0.045,636 +"Zhu2013",75,15,4,"ratchet",0.046,635 +"Zhu2013",75,15,4,"drift_exact",0.029,629 +"Zhu2013",75,15,4,"drift_union",0.126,625 +"Zhu2013",75,15,8,"ratchet",0.088,629 +"Zhu2013",75,15,8,"drift_exact",0.066,629 +"Zhu2013",75,15,8,"drift_union",0.211,625 +"Zhu2013",75,15,16,"ratchet",0.166,629 +"Zhu2013",75,15,16,"drift_exact",0.117,628 +"Zhu2013",75,15,16,"drift_union",0.413,624 +"Zhu2013",75,16,1,"ratchet",0.013,629 +"Zhu2013",75,16,1,"drift_exact",0.013,628 +"Zhu2013",75,16,1,"drift_union",0.033,629 +"Zhu2013",75,16,2,"ratchet",0.025,629 +"Zhu2013",75,16,2,"drift_exact",0.019,628 +"Zhu2013",75,16,2,"drift_union",0.039,629 +"Zhu2013",75,16,4,"ratchet",0.048,629 +"Zhu2013",75,16,4,"drift_exact",0.032,628 +"Zhu2013",75,16,4,"drift_union",0.101,628 +"Zhu2013",75,16,8,"ratchet",0.082,629 +"Zhu2013",75,16,8,"drift_exact",0.067,628 +"Zhu2013",75,16,8,"drift_union",0.198,626 +"Zhu2013",75,16,16,"ratchet",0.156,628 +"Zhu2013",75,16,16,"drift_exact",0.126,628 +"Zhu2013",75,16,16,"drift_union",0.366,626 +"Zhu2013",75,17,1,"ratchet",0.013,629 +"Zhu2013",75,17,1,"drift_exact",0.016,629 +"Zhu2013",75,17,1,"drift_union",0.045,625 +"Zhu2013",75,17,2,"ratchet",0.021,629 +"Zhu2013",75,17,2,"drift_exact",0.022,629 +"Zhu2013",75,17,2,"drift_union",0.049,625 +"Zhu2013",75,17,4,"ratchet",0.04,629 +"Zhu2013",75,17,4,"drift_exact",0.035,629 +"Zhu2013",75,17,4,"drift_union",0.098,625 +"Zhu2013",75,17,8,"ratchet",0.074,629 +"Zhu2013",75,17,8,"drift_exact",0.068,627 +"Zhu2013",75,17,8,"drift_union",0.183,625 +"Zhu2013",75,17,16,"ratchet",0.144,629 +"Zhu2013",75,17,16,"drift_exact",0.132,625 +"Zhu2013",75,17,16,"drift_union",0.389,625 +"Zhu2013",75,18,1,"ratchet",0.021,630 +"Zhu2013",75,18,1,"drift_exact",0.014,628 +"Zhu2013",75,18,1,"drift_union",0.055,626 +"Zhu2013",75,18,2,"ratchet",0.034,626 +"Zhu2013",75,18,2,"drift_exact",0.018,628 +"Zhu2013",75,18,2,"drift_union",0.059,626 +"Zhu2013",75,18,4,"ratchet",0.055,626 +"Zhu2013",75,18,4,"drift_exact",0.031,628 +"Zhu2013",75,18,4,"drift_union",0.105,625 +"Zhu2013",75,18,8,"ratchet",0.092,626 +"Zhu2013",75,18,8,"drift_exact",0.057,627 +"Zhu2013",75,18,8,"drift_union",0.206,625 +"Zhu2013",75,18,16,"ratchet",0.165,625 +"Zhu2013",75,18,16,"drift_exact",0.118,627 +"Zhu2013",75,18,16,"drift_union",0.378,625 +"Zhu2013",75,19,1,"ratchet",0.015,626 +"Zhu2013",75,19,1,"drift_exact",0.011,627 +"Zhu2013",75,19,1,"drift_union",0.055,627 +"Zhu2013",75,19,2,"ratchet",0.023,626 +"Zhu2013",75,19,2,"drift_exact",0.02,627 +"Zhu2013",75,19,2,"drift_union",0.064,627 +"Zhu2013",75,19,4,"ratchet",0.041,626 +"Zhu2013",75,19,4,"drift_exact",0.036,625 +"Zhu2013",75,19,4,"drift_union",0.095,627 +"Zhu2013",75,19,8,"ratchet",0.08,625 +"Zhu2013",75,19,8,"drift_exact",0.072,625 +"Zhu2013",75,19,8,"drift_union",0.184,626 +"Zhu2013",75,19,16,"ratchet",0.154,625 +"Zhu2013",75,19,16,"drift_exact",0.131,625 +"Zhu2013",75,19,16,"drift_union",0.368,625 +"Zhu2013",75,20,1,"ratchet",0.013,630 +"Zhu2013",75,20,1,"drift_exact",0.009,630 +"Zhu2013",75,20,1,"drift_union",0.053,629 +"Zhu2013",75,20,2,"ratchet",0.021,630 +"Zhu2013",75,20,2,"drift_exact",0.014,630 +"Zhu2013",75,20,2,"drift_union",0.061,626 +"Zhu2013",75,20,4,"ratchet",0.04,627 +"Zhu2013",75,20,4,"drift_exact",0.026,630 +"Zhu2013",75,20,4,"drift_union",0.102,626 +"Zhu2013",75,20,8,"ratchet",0.076,627 +"Zhu2013",75,20,8,"drift_exact",0.053,627 +"Zhu2013",75,20,8,"drift_union",0.209,625 +"Zhu2013",75,20,16,"ratchet",0.156,627 +"Zhu2013",75,20,16,"drift_exact",0.103,627 +"Zhu2013",75,20,16,"drift_union",0.382,625 diff --git a/src/ts_drift.cpp b/src/ts_drift.cpp index 8d95d13fc..7af6a96a1 100644 --- a/src/ts_drift.cpp +++ b/src/ts_drift.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -16,6 +17,47 @@ namespace ts { +// Scan-accuracy census for the pure-EW drift candidate scorer (env +// TS_DRIFT_SCANCHK). Quantifies how far the deployed union-of-finals +// approximation (fitch_indirect_length_bounded, reads the incrementally +// maintained tree.final_) diverges from the EXACT directional insertion cost +// (fitch_indirect_length_cached over compute_insertion_edge_sets), the same +// exact quantity tbr_search migrated to in 2b299e4b. +// +// The union error is TWO-SIDED on the deployed path (empirically): it both +// over- and under-counts by up to tens of steps -- NOT the strict over-count +// the exactness gate measured with FRESH finals (dev/profiling/exactness-gate.md +// P2). The decision-relevant harm comes from the UNDER-counts via an +// optimizer's-curse effect: selecting argmin over (divided_length + u_cost) +// preferentially picks candidates whose u_cost < e_cost (the moves the estimator +// is most optimistic about), so the SELECTED move's error is biased negative -- +// a truly-expensive move can look cheap, pass the AFD gate, and be applied +// outside drift's intended drift envelope. The exact scan (error == 0) has no +// such bias. Recorded scores stay exact regardless (every accept is +// drift_full_rescore'd) -- this measures the scan/decision error, not a leak. +struct DriftEwCensus { + long long clips = 0; // EW drift clips the census examined + long long candidates = 0; // EW candidates scored (SPR + reroot) + long long overcount_cands = 0; // candidates where union > exact + long long overcount_sum = 0; // sum of (union - exact) over over-counts + int overcount_max = 0; // largest single (union - exact) + long long undercount_cands = 0; // candidates where union < exact (dangerous) + long long undercount_sum = 0; // sum of (exact - union) over under-counts + int undercount_max = 0; // largest single (exact - union) + long long select_flip = 0; // clips where union-best move != exact-best move + double select_regret_sum = 0.0; // sum over select_flip clips of exact-cost regret + long long env_violation = 0; // clips where union admits a move whose TRUE + // delta exceeds afd_limit (drift walks outside + // its intended AFD envelope) + // TS_IW_SCANCHK analog: predicted best_candidate vs post-apply full_rescore of + // the applied move. On the exact path this MUST be 0 (validates the wiring). + long long applied_checked = 0; + long long applied_mismatch = 0; + long long applied_under = 0; // applied moves truly WORSE than the scan said + // (optimizer's-curse signature) + double applied_maxdiff = 0.0; +}; + // --- Helpers (file-local, mirrored from ts_tbr.cpp) --- static double drift_full_rescore(TreeState& tree, const DataSet& ds) { @@ -320,7 +362,9 @@ static int drift_phase(TreeState& tree, const DataSet& ds, int afd_limit, double rfd_limit, int max_changes, std::mt19937& rng, ConstraintData* cd = nullptr, - const std::vector* sector_mask = nullptr) { + const std::vector* sector_mask = nullptr, + bool ew_exact = false, bool scanchk = false, + DriftEwCensus* census = nullptr) { bool constrained = cd && cd->active; if (constrained) update_constraint(tree, *cd); double score = drift_full_rescore(tree, ds); @@ -366,6 +410,19 @@ static int drift_phase(TreeState& tree, const DataSet& ds, static_cast(tree.n_node) * tree.total_words, 0); std::vector virtual_prelim(tree.total_words); + // Exact directional insertion edge sets for the pure-EW path, mirroring + // tbr_search (ts_tbr.cpp:1749). Computed once per clip; edge_set[below] is + // the exact edge set above node `below`, used by both the SPR and reroot + // scans. Caller-owned scratch, size-ensured (non-zeroing) and reused across + // clips. Only touched on the EW path when the exact scorer or the census is + // active; NA/IW drift never allocate these. + const int tw = tree.total_words; + const bool ew_path = !has_na && !use_iw; + const bool need_edge_set = ew_path && (ew_exact || scanchk); + std::vector edge_set_buf; + std::vector edge_set_up; + std::vector edge_set_pre; + // IW buffers std::vector divided_steps; std::vector iw_delta; @@ -442,6 +499,14 @@ static int drift_phase(TreeState& tree, const DataSet& ds, divided_length = score + delta - nx_cost; } + // Exact directional insertion edge sets for the pure-EW path (mirrors + // ts_tbr.cpp:1749): computed once per clip from the current clipped-tree + // downpass, then indexed by `below` in both the SPR and reroot scans. + if (need_edge_set) { + compute_insertion_edge_sets(tree, ds, edge_set_buf, + edge_set_up, edge_set_pre); + } + // Weighted scoring (IW or profile): precompute base score and deltas double base_iw = 0.0; if (use_iw) { @@ -473,6 +538,52 @@ static int drift_phase(TreeState& tree, const DataSet& ds, size_t clip_base = static_cast(clip_node) * tree.total_words; const uint64_t* clip_prelim = &tree.prelim[clip_base]; + // Per-clip census bests (only used under scanchk): the union-selected best + // move (cen_bc_union) with the EXACT cost of that same move (cen_e_at_union), + // and the exact-selected best move (cen_bc_exact). All are full candidate + // scores (divided_length + insertion cost). + double cen_bc_union = HUGE_VAL, cen_bc_exact = HUGE_VAL, + cen_e_at_union = HUGE_VAL; + + // Pure-EW candidate scorer: returns divided_length + insertion cost. When + // ew_exact, selects on the EXACT directional edge set (edge_set_buf[below]), + // matching tbr_search; otherwise the deployed union-of-finals approximation. + // Under scanchk it computes BOTH (unbounded -- byte-identical selection to + // the bounded form, since the bail only triggers once the value already + // exceeds the cutoff) and feeds the decision-flip census. `above` is only + // needed by the union scorer; the exact scorer indexes by `below`. + auto score_ew = [&](const uint64_t* prelim, int above, int below, + int ew_cutoff) -> double { + if (!scanchk) { + int extra = ew_exact + ? fitch_indirect_length_cached( + prelim, &edge_set_buf[static_cast(below) * tw], + ds, ew_cutoff) + : fitch_indirect_length_bounded(prelim, tree, ds, + above, below, ew_cutoff); + return divided_length + extra; + } + int u_cost = fitch_indirect_length_bounded(prelim, tree, ds, + above, below, INT_MAX); + int e_cost = fitch_indirect_length_cached( + prelim, &edge_set_buf[static_cast(below) * tw], ds, INT_MAX); + ++census->candidates; + int over = u_cost - e_cost; // union - exact; TWO-SIDED on the deployed path + if (over > 0) { + ++census->overcount_cands; + census->overcount_sum += over; + if (over > census->overcount_max) census->overcount_max = over; + } else if (over < 0) { + ++census->undercount_cands; + census->undercount_sum += (-over); + if (-over > census->undercount_max) census->undercount_max = -over; + } + double cu = divided_length + u_cost, ce = divided_length + e_cost; + if (cu < cen_bc_union) { cen_bc_union = cu; cen_e_at_union = ce; } + if (ce < cen_bc_exact) { cen_bc_exact = ce; } + return divided_length + (ew_exact ? e_cost : u_cost); + }; + // SPR candidates (bounded to skip losing positions early) for (auto& [above, below] : main_edges) { if (above == nz && below == ns) continue; @@ -501,9 +612,7 @@ static int drift_phase(TreeState& tree, const DataSet& ds, } else { int ew_cutoff = (best_candidate < HUGE_VAL) ? static_cast(best_candidate - divided_length) : INT_MAX; - candidate = divided_length + - fitch_indirect_length_bounded(clip_prelim, tree, ds, - above, below, ew_cutoff); + candidate = score_ew(clip_prelim, above, below, ew_cutoff); } if (candidate < best_candidate) { best_candidate = candidate; @@ -559,9 +668,7 @@ static int drift_phase(TreeState& tree, const DataSet& ds, int ew_cutoff = (best_candidate < HUGE_VAL) ? static_cast(best_candidate - divided_length) : INT_MAX; - candidate = divided_length + - fitch_indirect_length_bounded(virtual_prelim.data(), tree, ds, - above, below, ew_cutoff); + candidate = score_ew(virtual_prelim.data(), above, below, ew_cutoff); } if (candidate < best_candidate) { best_candidate = candidate; @@ -574,6 +681,23 @@ static int drift_phase(TreeState& tree, const DataSet& ds, } } + // Census: per-clip decision accounting. select_flip counts clips where the + // union-chosen move is EXACT-suboptimal (the exact scorer would pick a + // strictly better move). env_violation counts clips where the union scan + // GATES IN its chosen move (union delta within the AFD limit) but that + // move's TRUE cost exceeds the AFD limit -- drift then walks outside its + // intended drift envelope, the direct consequence of the under-count. + if (scanchk && cen_bc_union < HUGE_VAL) { + ++census->clips; + if (cen_e_at_union > cen_bc_exact + eps) { + ++census->select_flip; + census->select_regret_sum += (cen_e_at_union - cen_bc_exact); + } + bool union_gated_in = (cen_bc_union - score) <= afd_limit + eps; + bool true_out_of_env = (cen_e_at_union - score) > afd_limit + eps; + if (union_gated_in && true_out_of_env) ++census->env_violation; + } + // --- Phase 2: Restore and decide --- tree.restore_prealloc_undo(); tree.spr_unclip(); @@ -622,6 +746,23 @@ static int drift_phase(TreeState& tree, const DataSet& ds, } } + // TS_IW_SCANCHK analog: the move has been applied and validated; compare the + // scan's predicted best_candidate against the authoritative full_rescore. + // One extra rescore per applied clip (diagnostic only); the branch logic + // below recomputes score idempotently. + if (scanchk) { + tree.build_postorder(); + double applied_exact = drift_full_rescore(tree, ds); + ++census->applied_checked; + double signed_err = applied_exact - best_candidate; // >0 => truth worse + double d = std::fabs(signed_err); + if (d > 1e-6) { + ++census->applied_mismatch; + if (signed_err > 0) ++census->applied_under; // optimizer's-curse signature + if (d > census->applied_maxdiff) census->applied_maxdiff = d; + } + } + int n_before = n_accepted; if (delta_score < -eps) { @@ -739,6 +880,16 @@ DriftResult drift_search(TreeState& tree, const DataSet& ds, int total_drift_moves = 0; int total_tbr_moves = 0; + // Env flags read ONCE per drift_search (never per candidate: std::getenv is + // ~2.4us on ucrt and would confound matched-wall timing -- see memory node + // getenv-ucrt-cost). TS_DRIFT_EXACT routes the pure-EW drift scorer to the + // exact directional edge set (opt-in; union-of-finals remains the default so + // committed behaviour is unchanged until the matched-wall gate clears it). + // TS_DRIFT_SCANCHK enables the divergence census below. + const bool ew_exact = std::getenv("TS_DRIFT_EXACT") != nullptr; + const bool scanchk = std::getenv("TS_DRIFT_SCANCHK") != nullptr; + DriftEwCensus census; + // Seed RNG (from R in serial mode, from thread-local in parallel mode) std::mt19937 rng = ts::make_rng(); @@ -756,7 +907,9 @@ DriftResult drift_search(TreeState& tree, const DataSet& ds, // Suboptimal drift phase int drift_moves = drift_phase(tree, ds, params.afd_limit, params.rfd_limit, - max_drift_changes, rng, cd, sector_mask); + max_drift_changes, rng, cd, sector_mask, + ew_exact, scanchk, + scanchk ? &census : nullptr); total_drift_moves += drift_moves; } else { // Equal-score drift phase @@ -801,6 +954,36 @@ DriftResult drift_search(TreeState& tree, const DataSet& ds, tree.build_postorder(); drift_full_rescore(tree, ds); + if (scanchk) { + // over/under: two-sided per-candidate scan error vs the exact directional + // cost. select_flip/env_violation: per-clip decision harm on the union + // path (property of the candidate set; on the exact path they measure what + // union WOULD have done). applied_mismatch/under: scan-vs-full_rescore of + // the APPLIED move -- MUST be 0 on the exact path (ew_exact=1), validating + // the wiring; on the union path applied_under exposes the optimizer's curse. + REprintf("DRIFT-SCANCHK ew_exact=%d clips=%lld cands=%lld " + "over=%lld(%.1f%%,max%d) under=%lld(%.1f%%,max%d) " + "select_flip=%lld(%.1f%%) regret=%.0f env_violation=%lld(%.1f%%) " + "applied=%lld mism=%lld under=%lld maxdiff=%.1f\n", + ew_exact ? 1 : 0, + census.clips, census.candidates, + census.overcount_cands, + census.candidates + ? 100.0 * census.overcount_cands / census.candidates : 0.0, + census.overcount_max, + census.undercount_cands, + census.candidates + ? 100.0 * census.undercount_cands / census.candidates : 0.0, + census.undercount_max, + census.select_flip, + census.clips ? 100.0 * census.select_flip / census.clips : 0.0, + census.select_regret_sum, + census.env_violation, + census.clips ? 100.0 * census.env_violation / census.clips : 0.0, + census.applied_checked, census.applied_mismatch, + census.applied_under, census.applied_maxdiff); + } + return DriftResult{best_score, params.n_cycles, total_drift_moves, total_tbr_moves}; } diff --git a/src/ts_search.cpp b/src/ts_search.cpp index b4097f2ab..03ca76398 100644 --- a/src/ts_search.cpp +++ b/src/ts_search.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -212,6 +213,24 @@ SearchResult spr_search(TreeState& tree, const DataSet& ds, int maxHits, if (ds.blocks[b].has_inapplicable) { has_na = true; break; } } + // Pure-EW SPR uses the EXACT directional edge set (like tbr_search, 2b299e4b), + // NOT the union-of-finals approximation. Union OVER-counts, inflating + // best_candidate so the `dominated` gate (line ~379) triggers too readily and + // HIDES improving moves -- and spr_search is a pure hill-climb with no + // exact-TBR phase to recover, so union converges catastrophically short + // (Zhu2013 best-of-25 starts 650 vs exact 625; ~25-70 steps, systematic; exact + // is also faster per start -- dev/profiling/drift-exactness-gate.md). So exact + // is the DEFAULT here; `TS_SPR_UNION` restores the old union scorer (kill + // switch). Read once (getenv ~2.4us on ucrt). Only reaches EW spr_search + // (NA/IW keep their own scorers below); production use is the sprFirst warmup + // (OFF in all presets) + the standalone ts_spr_search binding. + const bool ew_exact = std::getenv("TS_SPR_UNION") == nullptr; + const int tw = tree.total_words; + const bool ew_path = !has_na && !use_iw; + const bool need_edge_set = ew_path && ew_exact; + std::vector edge_set_buf, edge_set_up; + std::vector edge_set_pre; + // Seed RNG (from R in serial mode, from thread-local in parallel mode) std::mt19937 rng = ts::make_rng(); @@ -306,6 +325,13 @@ SearchResult spr_search(TreeState& tree, const DataSet& ds, int maxHits, divided_length = best_score + delta - nx_cost; } + // Exact directional insertion edge sets for the pure-EW path (mirrors + // ts_tbr.cpp:1749); indexed by `below` in the candidate scan below. + if (need_edge_set) { + compute_insertion_edge_sets(tree, ds, edge_set_buf, + edge_set_up, edge_set_pre); + } + const uint64_t* clip_prelim = &tree.prelim[static_cast(clip_node) * tree.total_words]; @@ -362,8 +388,12 @@ SearchResult spr_search(TreeState& tree, const DataSet& ds, int maxHits, int cutoff = (best_candidate < HUGE_VAL) ? static_cast(best_candidate - divided_length + 1) : INT_MAX; - int extra = fitch_indirect_length_bounded( - clip_prelim, tree, ds, above, below, cutoff); + int extra = ew_exact + ? fitch_indirect_length_cached( + clip_prelim, &edge_set_buf[static_cast(below) * tw], + ds, cutoff) + : fitch_indirect_length_bounded( + clip_prelim, tree, ds, above, below, cutoff); candidate_score = divided_length + extra; } ++n_iterations;